2024-06-18T02:28:16.881 INFO:root:teuthology version: 0.0.1.dev325+g8e97141 2024-06-18T02:28:16.882 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2024-06-17_21:08:03-orch-squid-distro-default-smithi/7760446 --name teuthology-2024-06-17_21:08:03-orch-squid-distro-default-smithi --description orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} -- /home/teuthworker/archive/teuthology-2024-06-17_21:08:03-orch-squid-distro-default-smithi/7760446/orig.config.yaml 2024-06-18T02:28:16.924 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-06-18T02:28:17.008 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-06-17_21:08:03-orch-squid-distro-default-smithi/7760446 branch: squid description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: ceph-qa@ceph.com first_in_suite: false job_id: '7760446' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-06-17_21:08:03-orch-squid-distro-default-smithi no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: 3781e3a592387d16e8020f5db99cdee4198f6d00 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 3781e3a592387d16e8020f5db99cdee4198f6d00 workunit: branch: squid sha1: f18f3f1cbd11e4a41c557e616d172f24deb91473 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 4712 sha1: 3781e3a592387d16e8020f5db99cdee4198f6d00 sleep_before_teardown: 0 subset: 61/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_f18f3f1cbd11e4a41c557e616d172f24deb91473/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: f18f3f1cbd11e4a41c557e616d172f24deb91473 targets: smithi163.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJB690bDQhZ5LhbhEyPvyBxIljgc80QRm5h1F80d3WNArN6Fu+Zd9fxfPNOLDN67KIjEW/YqwpU6TIr3pCpp3Fw= smithi186.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBM/mgQChdgKCVa+nQ1Ks1CMcsP8Sd94fL0vv2g+7lI+8Ljto170s3Ix26zocMDpGUWvOStJb5gx6XyXiaCtevg= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest false --force - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 8e9714173de9e92c97e8ef1045d333e96b793454 timestamp: 2024-06-17_21:08:03 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2698841 2024-06-18T02:28:17.008 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_f18f3f1cbd11e4a41c557e616d172f24deb91473/qa; will attempt to use it 2024-06-18T02:28:17.009 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_f18f3f1cbd11e4a41c557e616d172f24deb91473/qa/tasks 2024-06-18T02:28:17.009 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-06-18T02:28:17.010 INFO:teuthology.task.internal:Checking packages... 2024-06-18T02:28:17.029 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '3781e3a592387d16e8020f5db99cdee4198f6d00' 2024-06-18T02:28:17.029 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-06-18T02:28:17.029 INFO:teuthology.packaging:ref: None 2024-06-18T02:28:17.029 INFO:teuthology.packaging:tag: None 2024-06-18T02:28:17.029 INFO:teuthology.packaging:branch: squid 2024-06-18T02:28:17.029 INFO:teuthology.packaging:sha1: 3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T02:28:17.030 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-06-18T02:28:17.417 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3145.gf18f3f1c 2024-06-18T02:28:17.418 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-06-18T02:28:17.425 INFO:teuthology.task.internal:no buildpackages task found 2024-06-18T02:28:17.425 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-06-18T02:28:17.464 INFO:teuthology.task.internal:Saving configuration 2024-06-18T02:28:17.480 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-06-18T02:28:17.486 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-06-18T02:28:17.510 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi163.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-06-17_21:08:03-orch-squid-distro-default-smithi/7760446', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-06-18 02:22:33.665007', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJB690bDQhZ5LhbhEyPvyBxIljgc80QRm5h1F80d3WNArN6Fu+Zd9fxfPNOLDN67KIjEW/YqwpU6TIr3pCpp3Fw='} 2024-06-18T02:28:17.534 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi186.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-06-17_21:08:03-orch-squid-distro-default-smithi/7760446', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-06-18 02:22:33.666703', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBM/mgQChdgKCVa+nQ1Ks1CMcsP8Sd94fL0vv2g+7lI+8Ljto170s3Ix26zocMDpGUWvOStJb5gx6XyXiaCtevg='} 2024-06-18T02:28:17.534 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-06-18T02:28:17.540 INFO:teuthology.task.internal:roles: ubuntu@smithi163.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-06-18T02:28:17.540 INFO:teuthology.task.internal:roles: ubuntu@smithi186.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-06-18T02:28:17.540 INFO:teuthology.run_tasks:Running task console_log... 2024-06-18T02:28:17.619 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f760695fac0>, signals=[15]) 2024-06-18T02:28:17.619 INFO:teuthology.run_tasks:Running task internal.connect... 2024-06-18T02:28:17.625 INFO:teuthology.task.internal:Opening connections... 2024-06-18T02:28:17.625 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi163.front.sepia.ceph.com 2024-06-18T02:28:17.626 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-18T02:28:17.704 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi186.front.sepia.ceph.com 2024-06-18T02:28:17.704 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi186.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-18T02:28:17.777 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-06-18T02:28:17.783 DEBUG:teuthology.orchestra.run.smithi163:> uname -m 2024-06-18T02:28:17.801 INFO:teuthology.orchestra.run.smithi163.stdout:x86_64 2024-06-18T02:28:17.801 DEBUG:teuthology.orchestra.run.smithi163:> cat /etc/os-release 2024-06-18T02:28:17.857 INFO:teuthology.orchestra.run.smithi163.stdout:NAME="CentOS Stream" 2024-06-18T02:28:17.857 INFO:teuthology.orchestra.run.smithi163.stdout:VERSION="9" 2024-06-18T02:28:17.857 INFO:teuthology.orchestra.run.smithi163.stdout:ID="centos" 2024-06-18T02:28:17.858 INFO:teuthology.orchestra.run.smithi163.stdout:ID_LIKE="rhel fedora" 2024-06-18T02:28:17.858 INFO:teuthology.orchestra.run.smithi163.stdout:VERSION_ID="9" 2024-06-18T02:28:17.858 INFO:teuthology.orchestra.run.smithi163.stdout:PLATFORM_ID="platform:el9" 2024-06-18T02:28:17.858 INFO:teuthology.orchestra.run.smithi163.stdout:PRETTY_NAME="CentOS Stream 9" 2024-06-18T02:28:17.858 INFO:teuthology.orchestra.run.smithi163.stdout:ANSI_COLOR="0;31" 2024-06-18T02:28:17.858 INFO:teuthology.orchestra.run.smithi163.stdout:LOGO="fedora-logo-icon" 2024-06-18T02:28:17.858 INFO:teuthology.orchestra.run.smithi163.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-06-18T02:28:17.858 INFO:teuthology.orchestra.run.smithi163.stdout:HOME_URL="https://centos.org/" 2024-06-18T02:28:17.858 INFO:teuthology.orchestra.run.smithi163.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-06-18T02:28:17.858 INFO:teuthology.orchestra.run.smithi163.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-06-18T02:28:17.858 INFO:teuthology.orchestra.run.smithi163.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-06-18T02:28:17.859 INFO:teuthology.lock.ops:Updating smithi163.front.sepia.ceph.com on lock server 2024-06-18T02:28:17.874 DEBUG:teuthology.orchestra.run.smithi186:> uname -m 2024-06-18T02:28:17.892 INFO:teuthology.orchestra.run.smithi186.stdout:x86_64 2024-06-18T02:28:17.892 DEBUG:teuthology.orchestra.run.smithi186:> cat /etc/os-release 2024-06-18T02:28:17.949 INFO:teuthology.orchestra.run.smithi186.stdout:NAME="CentOS Stream" 2024-06-18T02:28:17.949 INFO:teuthology.orchestra.run.smithi186.stdout:VERSION="9" 2024-06-18T02:28:17.949 INFO:teuthology.orchestra.run.smithi186.stdout:ID="centos" 2024-06-18T02:28:17.949 INFO:teuthology.orchestra.run.smithi186.stdout:ID_LIKE="rhel fedora" 2024-06-18T02:28:17.950 INFO:teuthology.orchestra.run.smithi186.stdout:VERSION_ID="9" 2024-06-18T02:28:17.950 INFO:teuthology.orchestra.run.smithi186.stdout:PLATFORM_ID="platform:el9" 2024-06-18T02:28:17.950 INFO:teuthology.orchestra.run.smithi186.stdout:PRETTY_NAME="CentOS Stream 9" 2024-06-18T02:28:17.950 INFO:teuthology.orchestra.run.smithi186.stdout:ANSI_COLOR="0;31" 2024-06-18T02:28:17.950 INFO:teuthology.orchestra.run.smithi186.stdout:LOGO="fedora-logo-icon" 2024-06-18T02:28:17.950 INFO:teuthology.orchestra.run.smithi186.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-06-18T02:28:17.950 INFO:teuthology.orchestra.run.smithi186.stdout:HOME_URL="https://centos.org/" 2024-06-18T02:28:17.950 INFO:teuthology.orchestra.run.smithi186.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-06-18T02:28:17.950 INFO:teuthology.orchestra.run.smithi186.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-06-18T02:28:17.950 INFO:teuthology.orchestra.run.smithi186.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-06-18T02:28:17.951 INFO:teuthology.lock.ops:Updating smithi186.front.sepia.ceph.com on lock server 2024-06-18T02:28:17.965 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-06-18T02:28:17.974 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-06-18T02:28:17.989 INFO:teuthology.task.internal:Checking for old test directory... 2024-06-18T02:28:17.989 DEBUG:teuthology.orchestra.run.smithi163:> test '!' -e /home/ubuntu/cephtest 2024-06-18T02:28:17.992 DEBUG:teuthology.orchestra.run.smithi186:> test '!' -e /home/ubuntu/cephtest 2024-06-18T02:28:18.008 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-06-18T02:28:18.027 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-06-18T02:28:18.027 DEBUG:teuthology.orchestra.run.smithi163:> test -z $(ls -A /var/lib/ceph) 2024-06-18T02:28:18.048 DEBUG:teuthology.orchestra.run.smithi186:> test -z $(ls -A /var/lib/ceph) 2024-06-18T02:28:18.141 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-06-18T02:28:18.192 INFO:teuthology.run_tasks:Running task kernel... 2024-06-18T02:28:18.205 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-06-18T02:28:18.206 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-06-18T02:28:18.206 DEBUG:teuthology.orchestra.run.smithi163:> test -f /run/.containerenv -o -f /.dockerenv 2024-06-18T02:28:18.206 DEBUG:teuthology.orchestra.run.smithi186:> test -f /run/.containerenv -o -f /.dockerenv 2024-06-18T02:28:18.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-18T02:28:18.222 DEBUG:teuthology.orchestra.run.smithi186:> uname -r 2024-06-18T02:28:18.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-18T02:28:18.223 DEBUG:teuthology.orchestra.run.smithi163:> uname -r 2024-06-18T02:28:18.278 INFO:teuthology.orchestra.run.smithi186.stdout:5.14.0-437.el9.x86_64 2024-06-18T02:28:18.278 INFO:teuthology.task.kernel:Running kernel on smithi186: 5.14.0-437.el9.x86_64 2024-06-18T02:28:18.278 DEBUG:teuthology.orchestra.run.smithi186:> sudo yum install -y kernel 2024-06-18T02:28:18.281 INFO:teuthology.orchestra.run.smithi163.stdout:5.14.0-437.el9.x86_64 2024-06-18T02:28:18.281 INFO:teuthology.task.kernel:Running kernel on smithi163: 5.14.0-437.el9.x86_64 2024-06-18T02:28:18.281 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum install -y kernel 2024-06-18T02:28:22.044 INFO:teuthology.orchestra.run.smithi163.stdout:CentOS Stream 9 - BaseOS 3.9 MB/s | 8.1 MB 00:02 2024-06-18T02:28:24.299 INFO:teuthology.orchestra.run.smithi186.stdout:CentOS Stream 9 - BaseOS 1.9 MB/s | 8.1 MB 00:04 2024-06-18T02:28:27.513 INFO:teuthology.orchestra.run.smithi163.stdout:CentOS Stream 9 - AppStream 4.5 MB/s | 20 MB 00:04 2024-06-18T02:28:33.507 INFO:teuthology.orchestra.run.smithi186.stdout:CentOS Stream 9 - AppStream 2.4 MB/s | 20 MB 00:08 2024-06-18T02:28:34.195 INFO:teuthology.orchestra.run.smithi163.stdout:CentOS Stream 9 - CRB 2.2 MB/s | 6.3 MB 00:02 2024-06-18T02:28:36.020 INFO:teuthology.orchestra.run.smithi163.stdout:CentOS Stream 9 - Extras packages 34 kB/s | 17 kB 00:00 2024-06-18T02:28:36.882 INFO:teuthology.orchestra.run.smithi163.stdout:Extra Packages for Enterprise Linux 30 MB/s | 21 MB 00:00 2024-06-18T02:28:39.346 INFO:teuthology.orchestra.run.smithi186.stdout:CentOS Stream 9 - CRB 3.0 MB/s | 6.3 MB 00:02 2024-06-18T02:28:40.950 INFO:teuthology.orchestra.run.smithi186.stdout:CentOS Stream 9 - Extras packages 64 kB/s | 17 kB 00:00 2024-06-18T02:28:41.979 INFO:teuthology.orchestra.run.smithi186.stdout:Extra Packages for Enterprise Linux 24 MB/s | 21 MB 00:00 2024-06-18T02:28:42.222 INFO:teuthology.orchestra.run.smithi163.stdout:lab-extras 19 kB/s | 1.7 kB 00:00 2024-06-18T02:28:44.328 INFO:teuthology.orchestra.run.smithi163.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-06-18T02:28:44.328 INFO:teuthology.orchestra.run.smithi163.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-06-18T02:28:44.407 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2024-06-18T02:28:44.418 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2024-06-18T02:28:44.418 INFO:teuthology.orchestra.run.smithi163.stdout: Package Architecture Version Repository Size 2024-06-18T02:28:44.418 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2024-06-18T02:28:44.418 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2024-06-18T02:28:44.419 INFO:teuthology.orchestra.run.smithi163.stdout: kernel x86_64 5.14.0-460.el9 baseos 1.1 M 2024-06-18T02:28:44.419 INFO:teuthology.orchestra.run.smithi163.stdout:Installing dependencies: 2024-06-18T02:28:44.419 INFO:teuthology.orchestra.run.smithi163.stdout: kernel-core x86_64 5.14.0-460.el9 baseos 16 M 2024-06-18T02:28:44.419 INFO:teuthology.orchestra.run.smithi163.stdout: kernel-modules x86_64 5.14.0-460.el9 baseos 35 M 2024-06-18T02:28:44.419 INFO:teuthology.orchestra.run.smithi163.stdout: kernel-modules-core x86_64 5.14.0-460.el9 baseos 29 M 2024-06-18T02:28:44.419 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:28:44.419 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2024-06-18T02:28:44.419 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2024-06-18T02:28:44.419 INFO:teuthology.orchestra.run.smithi163.stdout:Install 4 Packages 2024-06-18T02:28:44.419 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:28:44.419 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 81 M 2024-06-18T02:28:44.419 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 124 M 2024-06-18T02:28:44.420 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2024-06-18T02:28:44.639 INFO:teuthology.orchestra.run.smithi163.stdout:(1/4): kernel-5.14.0-460.el9.x86_64.rpm 5.7 MB/s | 1.1 MB 00:00 2024-06-18T02:28:45.040 INFO:teuthology.orchestra.run.smithi163.stdout:(2/4): kernel-core-5.14.0-460.el9.x86_64.rpm 28 MB/s | 16 MB 00:00 2024-06-18T02:28:45.407 INFO:teuthology.orchestra.run.smithi163.stdout:(3/4): kernel-modules-5.14.0-460.el9.x86_64.rpm 37 MB/s | 35 MB 00:00 2024-06-18T02:28:45.765 INFO:teuthology.orchestra.run.smithi163.stdout:(4/4): kernel-modules-core-5.14.0-460.el9.x86_6 26 MB/s | 29 MB 00:01 2024-06-18T02:28:45.765 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2024-06-18T02:28:45.766 INFO:teuthology.orchestra.run.smithi163.stdout:Total 60 MB/s | 81 MB 00:01 2024-06-18T02:28:46.472 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2024-06-18T02:28:46.817 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2024-06-18T02:28:46.817 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2024-06-18T02:28:47.202 INFO:teuthology.orchestra.run.smithi186.stdout:lab-extras 16 kB/s | 1.7 kB 00:00 2024-06-18T02:28:47.811 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2024-06-18T02:28:47.811 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2024-06-18T02:28:49.224 INFO:teuthology.orchestra.run.smithi186.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-06-18T02:28:49.224 INFO:teuthology.orchestra.run.smithi186.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-06-18T02:28:49.309 INFO:teuthology.orchestra.run.smithi186.stdout:Dependencies resolved. 2024-06-18T02:28:49.315 INFO:teuthology.orchestra.run.smithi186.stdout:================================================================================ 2024-06-18T02:28:49.316 INFO:teuthology.orchestra.run.smithi186.stdout: Package Architecture Version Repository Size 2024-06-18T02:28:49.316 INFO:teuthology.orchestra.run.smithi186.stdout:================================================================================ 2024-06-18T02:28:49.316 INFO:teuthology.orchestra.run.smithi186.stdout:Installing: 2024-06-18T02:28:49.316 INFO:teuthology.orchestra.run.smithi186.stdout: kernel x86_64 5.14.0-460.el9 baseos 1.1 M 2024-06-18T02:28:49.316 INFO:teuthology.orchestra.run.smithi186.stdout:Installing dependencies: 2024-06-18T02:28:49.316 INFO:teuthology.orchestra.run.smithi186.stdout: kernel-core x86_64 5.14.0-460.el9 baseos 16 M 2024-06-18T02:28:49.316 INFO:teuthology.orchestra.run.smithi186.stdout: kernel-modules x86_64 5.14.0-460.el9 baseos 35 M 2024-06-18T02:28:49.316 INFO:teuthology.orchestra.run.smithi186.stdout: kernel-modules-core x86_64 5.14.0-460.el9 baseos 29 M 2024-06-18T02:28:49.316 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-06-18T02:28:49.316 INFO:teuthology.orchestra.run.smithi186.stdout:Transaction Summary 2024-06-18T02:28:49.316 INFO:teuthology.orchestra.run.smithi186.stdout:================================================================================ 2024-06-18T02:28:49.316 INFO:teuthology.orchestra.run.smithi186.stdout:Install 4 Packages 2024-06-18T02:28:49.316 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-06-18T02:28:49.317 INFO:teuthology.orchestra.run.smithi186.stdout:Total download size: 81 M 2024-06-18T02:28:49.317 INFO:teuthology.orchestra.run.smithi186.stdout:Installed size: 124 M 2024-06-18T02:28:49.317 INFO:teuthology.orchestra.run.smithi186.stdout:Downloading Packages: 2024-06-18T02:28:50.226 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2024-06-18T02:28:50.244 INFO:teuthology.orchestra.run.smithi186.stdout:(1/4): kernel-5.14.0-460.el9.x86_64.rpm 1.3 MB/s | 1.1 MB 00:00 2024-06-18T02:28:50.946 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : kernel-modules-core-5.14.0-460.el9.x86_64 1/4 2024-06-18T02:28:51.048 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : kernel-core-5.14.0-460.el9.x86_64 2/4 2024-06-18T02:28:53.734 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: kernel-core-5.14.0-460.el9.x86_64 2/4 2024-06-18T02:28:53.959 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : kernel-modules-5.14.0-460.el9.x86_64 3/4 2024-06-18T02:28:58.673 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: kernel-modules-5.14.0-460.el9.x86_64 3/4 2024-06-18T02:28:58.798 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : kernel-5.14.0-460.el9.x86_64 4/4 2024-06-18T02:29:03.555 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: kernel-modules-core-5.14.0-460.el9.x86_64 4/4 2024-06-18T02:29:06.793 INFO:teuthology.orchestra.run.smithi186.stdout:(2/4): kernel-core-5.14.0-460.el9.x86_64.rpm 952 kB/s | 16 MB 00:17 2024-06-18T02:29:23.900 INFO:teuthology.orchestra.run.smithi186.stdout:(3/4): kernel-modules-core-5.14.0-460.el9.x86_6 885 kB/s | 29 MB 00:33 2024-06-18T02:29:31.291 INFO:teuthology.orchestra.run.smithi186.stdout:(4/4): kernel-modules-5.14.0-460.el9.x86_64.rpm 852 kB/s | 35 MB 00:41 2024-06-18T02:29:31.291 INFO:teuthology.orchestra.run.smithi186.stdout:-------------------------------------------------------------------------------- 2024-06-18T02:29:31.291 INFO:teuthology.orchestra.run.smithi186.stdout:Total 1.9 MB/s | 81 MB 00:41 2024-06-18T02:29:32.000 INFO:teuthology.orchestra.run.smithi186.stdout:Running transaction check 2024-06-18T02:29:32.382 INFO:teuthology.orchestra.run.smithi186.stdout:Transaction check succeeded. 2024-06-18T02:29:32.383 INFO:teuthology.orchestra.run.smithi186.stdout:Running transaction test 2024-06-18T02:29:33.181 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: kernel-core-5.14.0-460.el9.x86_64 4/4 2024-06-18T02:29:33.222 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: kernel-modules-5.14.0-460.el9.x86_64 4/4 2024-06-18T02:29:33.332 INFO:teuthology.orchestra.run.smithi186.stdout:Transaction test succeeded. 2024-06-18T02:29:33.332 INFO:teuthology.orchestra.run.smithi186.stdout:Running transaction 2024-06-18T02:29:33.578 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: kernel-5.14.0-460.el9.x86_64 4/4 2024-06-18T02:29:33.578 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : kernel-5.14.0-460.el9.x86_64 1/4 2024-06-18T02:29:33.578 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : kernel-core-5.14.0-460.el9.x86_64 2/4 2024-06-18T02:29:33.578 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : kernel-modules-5.14.0-460.el9.x86_64 3/4 2024-06-18T02:29:34.376 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : kernel-modules-core-5.14.0-460.el9.x86_64 4/4 2024-06-18T02:29:34.376 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:29:34.376 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2024-06-18T02:29:34.376 INFO:teuthology.orchestra.run.smithi163.stdout: kernel-5.14.0-460.el9.x86_64 2024-06-18T02:29:34.376 INFO:teuthology.orchestra.run.smithi163.stdout: kernel-core-5.14.0-460.el9.x86_64 2024-06-18T02:29:34.377 INFO:teuthology.orchestra.run.smithi163.stdout: kernel-modules-5.14.0-460.el9.x86_64 2024-06-18T02:29:34.377 INFO:teuthology.orchestra.run.smithi163.stdout: kernel-modules-core-5.14.0-460.el9.x86_64 2024-06-18T02:29:34.377 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:29:34.377 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2024-06-18T02:29:34.732 DEBUG:teuthology.orchestra.run.smithi163:> rpm -q kernel | sort -rV | head -n 1 2024-06-18T02:29:34.814 INFO:teuthology.orchestra.run.smithi163.stdout:kernel-5.14.0-460.el9.x86_64 2024-06-18T02:29:34.814 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-460.el9.x86_64 2024-06-18T02:29:34.814 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-460.el9.x86_64 2024-06-18T02:29:34.814 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-06-18T02:29:34.814 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-06-18T02:29:34.814 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-06-18T02:29:34.814 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi163.front.sepia.ceph.com, path=None, version=distro) 2024-06-18T02:29:34.815 DEBUG:teuthology.orchestra.run.smithi163:> rpm -q kernel | sort -rV | head -n 1 2024-06-18T02:29:34.868 INFO:teuthology.orchestra.run.smithi163.stdout:kernel-5.14.0-460.el9.x86_64 2024-06-18T02:29:34.868 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-460.el9.x86_64 2024-06-18T02:29:34.869 DEBUG:teuthology.orchestra.run.smithi163:> sudo rpm -qi grub2-tools 2024-06-18T02:29:34.904 INFO:teuthology.orchestra.run.smithi163.stdout:Name : grub2-tools 2024-06-18T02:29:34.904 INFO:teuthology.orchestra.run.smithi163.stdout:Epoch : 1 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:Version : 2.06 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:Release : 68.el9 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:Architecture: x86_64 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:Group : Unspecified 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:Size : 8264254 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:License : GPLv3+ 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:Packager : builder@centos.org 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:Vendor : CentOS 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:URL : http://www.gnu.org/software/grub/ 2024-06-18T02:29:34.905 INFO:teuthology.orchestra.run.smithi163.stdout:Summary : Support tools for GRUB. 2024-06-18T02:29:34.906 INFO:teuthology.orchestra.run.smithi163.stdout:Description : 2024-06-18T02:29:34.906 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:29:34.906 INFO:teuthology.orchestra.run.smithi163.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-06-18T02:29:34.906 INFO:teuthology.orchestra.run.smithi163.stdout:customizable bootloader with modular architecture. It supports a rich 2024-06-18T02:29:34.906 INFO:teuthology.orchestra.run.smithi163.stdout:variety of kernel formats, file systems, computer architectures and 2024-06-18T02:29:34.906 INFO:teuthology.orchestra.run.smithi163.stdout:hardware devices. 2024-06-18T02:29:34.906 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:29:34.906 INFO:teuthology.orchestra.run.smithi163.stdout:This subpackage provides tools for support of all platforms. 2024-06-18T02:29:34.907 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-06-18T02:29:34.907 INFO:teuthology.task.kernel:Updating grub on smithi163 to boot 5.14.0-460.el9.x86_64 2024-06-18T02:29:34.907 DEBUG:teuthology.orchestra.run.smithi163:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-06-18T02:29:35.566 INFO:teuthology.orchestra.run.smithi163.stderr:Generating grub configuration file ... 2024-06-18T02:29:35.738 INFO:teuthology.orchestra.run.smithi186.stdout: Preparing : 1/1 2024-06-18T02:29:36.435 INFO:teuthology.orchestra.run.smithi186.stdout: Installing : kernel-modules-core-5.14.0-460.el9.x86_64 1/4 2024-06-18T02:29:36.532 INFO:teuthology.orchestra.run.smithi186.stdout: Installing : kernel-core-5.14.0-460.el9.x86_64 2/4 2024-06-18T02:29:36.963 INFO:teuthology.orchestra.run.smithi163.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-06-18T02:29:36.998 INFO:teuthology.orchestra.run.smithi163.stderr:done 2024-06-18T02:29:37.000 DEBUG:teuthology.orchestra.run.smithi163:> mktemp 2024-06-18T02:29:37.015 INFO:teuthology.orchestra.run.smithi163.stdout:/tmp/tmp.lwWpDauCOj 2024-06-18T02:29:37.016 DEBUG:teuthology.orchestra.run.smithi163:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.lwWpDauCOj 2024-06-18T02:29:37.082 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod 0666 /tmp/tmp.lwWpDauCOj 2024-06-18T02:29:37.227 DEBUG:teuthology.orchestra.remote:smithi163:/tmp/tmp.lwWpDauCOj is 6KB 2024-06-18T02:29:37.276 DEBUG:teuthology.orchestra.run.smithi163:> rm -fr /tmp/tmp.lwWpDauCOj 2024-06-18T02:29:37.291 DEBUG:teuthology.orchestra.run.smithi163:> sudo /bin/ls /boot/loader/entries || true 2024-06-18T02:29:37.351 INFO:teuthology.orchestra.run.smithi163.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-06-18T02:29:37.351 INFO:teuthology.orchestra.run.smithi163.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-06-18T02:29:37.352 INFO:teuthology.orchestra.run.smithi163.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-06-18T02:29:37.352 INFO:teuthology.orchestra.run.smithi163.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-460.el9.x86_64.conf 2024-06-18T02:29:37.353 DEBUG:teuthology.orchestra.run.smithi163:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-460.el9.x86_64 2024-06-18T02:29:37.506 DEBUG:teuthology.orchestra.run.smithi163:> sudo shutdown -r now 2024-06-18T02:29:39.204 INFO:teuthology.orchestra.run.smithi186.stdout: Running scriptlet: kernel-core-5.14.0-460.el9.x86_64 2/4 2024-06-18T02:29:39.349 INFO:teuthology.orchestra.run.smithi186.stdout: Installing : kernel-modules-5.14.0-460.el9.x86_64 3/4 2024-06-18T02:29:44.077 INFO:teuthology.orchestra.run.smithi186.stdout: Running scriptlet: kernel-modules-5.14.0-460.el9.x86_64 3/4 2024-06-18T02:29:44.131 INFO:teuthology.orchestra.run.smithi186.stdout: Installing : kernel-5.14.0-460.el9.x86_64 4/4 2024-06-18T02:29:48.897 INFO:teuthology.orchestra.run.smithi186.stdout: Running scriptlet: kernel-modules-core-5.14.0-460.el9.x86_64 4/4 2024-06-18T02:30:07.509 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-06-18T02:30:07.509 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi163.front.sepia.ceph.com' 2024-06-18T02:30:07.510 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-18T02:30:18.339 INFO:teuthology.orchestra.run.smithi186.stdout: Running scriptlet: kernel-core-5.14.0-460.el9.x86_64 4/4 2024-06-18T02:30:18.379 INFO:teuthology.orchestra.run.smithi186.stdout: Running scriptlet: kernel-modules-5.14.0-460.el9.x86_64 4/4 2024-06-18T02:30:18.678 INFO:teuthology.orchestra.run.smithi186.stdout: Running scriptlet: kernel-5.14.0-460.el9.x86_64 4/4 2024-06-18T02:30:18.678 INFO:teuthology.orchestra.run.smithi186.stdout: Verifying : kernel-5.14.0-460.el9.x86_64 1/4 2024-06-18T02:30:18.678 INFO:teuthology.orchestra.run.smithi186.stdout: Verifying : kernel-core-5.14.0-460.el9.x86_64 2/4 2024-06-18T02:30:18.678 INFO:teuthology.orchestra.run.smithi186.stdout: Verifying : kernel-modules-5.14.0-460.el9.x86_64 3/4 2024-06-18T02:30:19.458 INFO:teuthology.orchestra.run.smithi186.stdout: Verifying : kernel-modules-core-5.14.0-460.el9.x86_64 4/4 2024-06-18T02:30:19.458 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-06-18T02:30:19.458 INFO:teuthology.orchestra.run.smithi186.stdout:Installed: 2024-06-18T02:30:19.458 INFO:teuthology.orchestra.run.smithi186.stdout: kernel-5.14.0-460.el9.x86_64 2024-06-18T02:30:19.458 INFO:teuthology.orchestra.run.smithi186.stdout: kernel-core-5.14.0-460.el9.x86_64 2024-06-18T02:30:19.459 INFO:teuthology.orchestra.run.smithi186.stdout: kernel-modules-5.14.0-460.el9.x86_64 2024-06-18T02:30:19.459 INFO:teuthology.orchestra.run.smithi186.stdout: kernel-modules-core-5.14.0-460.el9.x86_64 2024-06-18T02:30:19.459 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-06-18T02:30:19.459 INFO:teuthology.orchestra.run.smithi186.stdout:Complete! 2024-06-18T02:30:19.813 DEBUG:teuthology.orchestra.run.smithi186:> rpm -q kernel | sort -rV | head -n 1 2024-06-18T02:30:19.893 INFO:teuthology.orchestra.run.smithi186.stdout:kernel-5.14.0-460.el9.x86_64 2024-06-18T02:30:19.893 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-460.el9.x86_64 2024-06-18T02:30:19.893 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-460.el9.x86_64 2024-06-18T02:30:19.893 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-06-18T02:30:19.893 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-06-18T02:30:19.894 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-06-18T02:30:19.894 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi186.front.sepia.ceph.com, path=None, version=distro) 2024-06-18T02:30:19.894 DEBUG:teuthology.orchestra.run.smithi186:> rpm -q kernel | sort -rV | head -n 1 2024-06-18T02:30:19.947 INFO:teuthology.orchestra.run.smithi186.stdout:kernel-5.14.0-460.el9.x86_64 2024-06-18T02:30:19.948 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-460.el9.x86_64 2024-06-18T02:30:19.948 DEBUG:teuthology.orchestra.run.smithi186:> sudo rpm -qi grub2-tools 2024-06-18T02:30:19.984 INFO:teuthology.orchestra.run.smithi186.stdout:Name : grub2-tools 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:Epoch : 1 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:Version : 2.06 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:Release : 68.el9 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:Architecture: x86_64 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:Group : Unspecified 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:Size : 8264254 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:License : GPLv3+ 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:Packager : builder@centos.org 2024-06-18T02:30:19.985 INFO:teuthology.orchestra.run.smithi186.stdout:Vendor : CentOS 2024-06-18T02:30:19.986 INFO:teuthology.orchestra.run.smithi186.stdout:URL : http://www.gnu.org/software/grub/ 2024-06-18T02:30:19.986 INFO:teuthology.orchestra.run.smithi186.stdout:Summary : Support tools for GRUB. 2024-06-18T02:30:19.986 INFO:teuthology.orchestra.run.smithi186.stdout:Description : 2024-06-18T02:30:19.986 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-06-18T02:30:19.986 INFO:teuthology.orchestra.run.smithi186.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-06-18T02:30:19.986 INFO:teuthology.orchestra.run.smithi186.stdout:customizable bootloader with modular architecture. It supports a rich 2024-06-18T02:30:19.986 INFO:teuthology.orchestra.run.smithi186.stdout:variety of kernel formats, file systems, computer architectures and 2024-06-18T02:30:19.986 INFO:teuthology.orchestra.run.smithi186.stdout:hardware devices. 2024-06-18T02:30:19.986 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-06-18T02:30:19.986 INFO:teuthology.orchestra.run.smithi186.stdout:This subpackage provides tools for support of all platforms. 2024-06-18T02:30:19.987 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-06-18T02:30:19.987 INFO:teuthology.task.kernel:Updating grub on smithi186 to boot 5.14.0-460.el9.x86_64 2024-06-18T02:30:19.987 DEBUG:teuthology.orchestra.run.smithi186:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-06-18T02:30:20.576 INFO:teuthology.orchestra.run.smithi186.stderr:Generating grub configuration file ... 2024-06-18T02:30:21.956 INFO:teuthology.orchestra.run.smithi186.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-06-18T02:30:21.984 INFO:teuthology.orchestra.run.smithi186.stderr:done 2024-06-18T02:30:21.986 DEBUG:teuthology.orchestra.run.smithi186:> mktemp 2024-06-18T02:30:22.007 INFO:teuthology.orchestra.run.smithi186.stdout:/tmp/tmp.LL66ynE7j1 2024-06-18T02:30:22.008 DEBUG:teuthology.orchestra.run.smithi186:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.LL66ynE7j1 2024-06-18T02:30:22.084 DEBUG:teuthology.orchestra.run.smithi186:> sudo chmod 0666 /tmp/tmp.LL66ynE7j1 2024-06-18T02:30:22.230 DEBUG:teuthology.orchestra.remote:smithi186:/tmp/tmp.LL66ynE7j1 is 6KB 2024-06-18T02:30:22.249 DEBUG:teuthology.orchestra.run.smithi186:> rm -fr /tmp/tmp.LL66ynE7j1 2024-06-18T02:30:22.263 DEBUG:teuthology.orchestra.run.smithi186:> sudo /bin/ls /boot/loader/entries || true 2024-06-18T02:30:22.329 INFO:teuthology.orchestra.run.smithi186.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-06-18T02:30:22.329 INFO:teuthology.orchestra.run.smithi186.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-06-18T02:30:22.329 INFO:teuthology.orchestra.run.smithi186.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-06-18T02:30:22.329 INFO:teuthology.orchestra.run.smithi186.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-460.el9.x86_64.conf 2024-06-18T02:30:22.330 DEBUG:teuthology.orchestra.run.smithi186:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-460.el9.x86_64 2024-06-18T02:30:22.488 DEBUG:teuthology.orchestra.run.smithi186:> sudo shutdown -r now 2024-06-18T02:30:26.079 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.163 2024-06-18T02:30:35.086 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi163.front.sepia.ceph.com' 2024-06-18T02:30:35.087 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-18T02:30:38.142 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.163 2024-06-18T02:30:50.158 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi163.front.sepia.ceph.com' 2024-06-18T02:30:50.159 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-18T02:30:52.493 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-06-18T02:30:52.494 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi186.front.sepia.ceph.com' 2024-06-18T02:30:52.494 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi186.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-18T02:30:53.214 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.163 2024-06-18T02:31:08.228 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi163.front.sepia.ceph.com' 2024-06-18T02:31:08.229 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-18T02:31:10.882 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.186 2024-06-18T02:31:19.890 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi186.front.sepia.ceph.com' 2024-06-18T02:31:19.891 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi186.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-18T02:31:22.946 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.186 2024-06-18T02:31:26.754 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.163 2024-06-18T02:31:34.954 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi186.front.sepia.ceph.com' 2024-06-18T02:31:34.955 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi186.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-18T02:31:44.762 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi163.front.sepia.ceph.com' 2024-06-18T02:31:44.763 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-18T02:31:45.038 DEBUG:teuthology.orchestra.run.smithi163:> true 2024-06-18T02:31:45.408 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi163.front.sepia.ceph.com' 2024-06-18T02:31:45.408 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-460.el9.x86_64"... 2024-06-18T02:31:45.408 DEBUG:teuthology.orchestra.run.smithi163:> uname -r 2024-06-18T02:31:45.423 INFO:teuthology.orchestra.run.smithi163.stdout:5.14.0-460.el9.x86_64 2024-06-18T02:31:45.423 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-460.el9.x86_64 vs 5.14.0-460.el9.x86_64 2024-06-18T02:31:45.423 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-06-18T02:31:45.423 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-06-18T02:31:46.424 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-06-18T02:31:46.424 DEBUG:teuthology.orchestra.run.smithi163:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-06-18T02:31:46.506 INFO:teuthology.orchestra.run.smithi163.stdout:ttyS1 2024-06-18T02:31:46.556 DEBUG:teuthology.parallel:result is None 2024-06-18T02:31:53.378 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.186 2024-06-18T02:32:08.380 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi186.front.sepia.ceph.com' 2024-06-18T02:32:08.380 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi186.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-18T02:32:08.635 DEBUG:teuthology.orchestra.run.smithi186:> true 2024-06-18T02:32:09.032 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi186.front.sepia.ceph.com' 2024-06-18T02:32:09.032 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-460.el9.x86_64"... 2024-06-18T02:32:09.032 DEBUG:teuthology.orchestra.run.smithi186:> uname -r 2024-06-18T02:32:09.050 INFO:teuthology.orchestra.run.smithi186.stdout:5.14.0-460.el9.x86_64 2024-06-18T02:32:09.050 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-460.el9.x86_64 vs 5.14.0-460.el9.x86_64 2024-06-18T02:32:09.050 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-06-18T02:32:09.050 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-06-18T02:32:10.051 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-06-18T02:32:10.051 DEBUG:teuthology.orchestra.run.smithi186:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-06-18T02:32:10.140 INFO:teuthology.orchestra.run.smithi186.stdout:ttyS1 2024-06-18T02:32:10.193 DEBUG:teuthology.parallel:result is None 2024-06-18T02:32:10.194 INFO:teuthology.run_tasks:Running task internal.base... 2024-06-18T02:32:10.201 INFO:teuthology.task.internal:Creating test directory... 2024-06-18T02:32:10.201 DEBUG:teuthology.orchestra.run.smithi163:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-06-18T02:32:10.204 DEBUG:teuthology.orchestra.run.smithi186:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-06-18T02:32:10.221 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-06-18T02:32:10.231 INFO:teuthology.run_tasks:Running task internal.archive... 2024-06-18T02:32:10.248 INFO:teuthology.task.internal:Creating archive directory... 2024-06-18T02:32:10.248 DEBUG:teuthology.orchestra.run.smithi163:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-06-18T02:32:10.264 DEBUG:teuthology.orchestra.run.smithi186:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-06-18T02:32:10.373 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-06-18T02:32:10.382 INFO:teuthology.task.internal:Enabling coredump saving... 2024-06-18T02:32:10.382 DEBUG:teuthology.orchestra.run.smithi163:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-06-18T02:32:10.385 DEBUG:teuthology.orchestra.run.smithi186:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-06-18T02:32:10.413 INFO:teuthology.orchestra.run.smithi163.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-18T02:32:10.416 INFO:teuthology.orchestra.run.smithi186.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-18T02:32:10.425 INFO:teuthology.orchestra.run.smithi163.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-18T02:32:10.426 INFO:teuthology.orchestra.run.smithi186.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-18T02:32:10.428 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-06-18T02:32:10.436 INFO:teuthology.task.internal:Configuring sudo... 2024-06-18T02:32:10.436 DEBUG:teuthology.orchestra.run.smithi163:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-06-18T02:32:10.469 DEBUG:teuthology.orchestra.run.smithi186:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-06-18T02:32:10.495 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-06-18T02:32:10.506 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-06-18T02:32:10.506 DEBUG:teuthology.orchestra.run.smithi163:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-06-18T02:32:10.538 DEBUG:teuthology.orchestra.run.smithi186:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-06-18T02:32:10.553 DEBUG:teuthology.orchestra.run.smithi163:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-18T02:32:10.639 DEBUG:teuthology.orchestra.run.smithi163:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-18T02:32:10.686 DEBUG:teuthology.orchestra.run.smithi163:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-18T02:32:10.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-18T02:32:10.812 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2024-06-18T02:32:10.812 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-06-18T02:32:10.878 DEBUG:teuthology.orchestra.run.smithi186:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-18T02:32:10.923 DEBUG:teuthology.orchestra.run.smithi186:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-18T02:32:10.964 DEBUG:teuthology.orchestra.run.smithi186:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-18T02:32:11.044 DEBUG:teuthology.orchestra.run.smithi186:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-18T02:32:11.088 DEBUG:teuthology.orchestra.run.smithi186:> set -ex 2024-06-18T02:32:11.088 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-06-18T02:32:11.154 DEBUG:teuthology.orchestra.run.smithi163:> sudo service rsyslog restart 2024-06-18T02:32:11.157 DEBUG:teuthology.orchestra.run.smithi186:> sudo service rsyslog restart 2024-06-18T02:32:11.217 INFO:teuthology.orchestra.run.smithi163.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-18T02:32:11.241 INFO:teuthology.orchestra.run.smithi186.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-18T02:32:11.581 INFO:teuthology.run_tasks:Running task internal.timer... 2024-06-18T02:32:11.589 INFO:teuthology.task.internal:Starting timer... 2024-06-18T02:32:11.589 INFO:teuthology.run_tasks:Running task pcp... 2024-06-18T02:32:11.618 INFO:teuthology.run_tasks:Running task selinux... 2024-06-18T02:32:11.635 DEBUG:teuthology.orchestra.run.smithi163:> sudo service auditd rotate 2024-06-18T02:32:11.712 INFO:teuthology.orchestra.run.smithi163.stdout:Rotating logs: 2024-06-18T02:32:11.714 DEBUG:teuthology.orchestra.run.smithi186:> sudo service auditd rotate 2024-06-18T02:32:11.763 INFO:teuthology.orchestra.run.smithi186.stdout:Rotating logs: 2024-06-18T02:32:11.764 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-06-18T02:32:11.764 DEBUG:teuthology.orchestra.run.smithi163:> /usr/sbin/getenforce 2024-06-18T02:32:11.792 INFO:teuthology.orchestra.run.smithi163.stdout:Permissive 2024-06-18T02:32:11.792 DEBUG:teuthology.orchestra.run.smithi186:> /usr/sbin/getenforce 2024-06-18T02:32:11.816 INFO:teuthology.orchestra.run.smithi186.stdout:Permissive 2024-06-18T02:32:11.816 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi163.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi186.front.sepia.ceph.com': 'permissive'} 2024-06-18T02:32:11.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-06-18T02:32:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-18T02:32:11.861 DEBUG:teuthology.orchestra.run.smithi186:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-06-18T02:32:11.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-18T02:32:11.887 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-06-18T02:32:11.887 DEBUG:teuthology.orchestra.run.smithi163:> sudo /usr/sbin/setenforce permissive 2024-06-18T02:32:11.928 DEBUG:teuthology.orchestra.run.smithi186:> sudo /usr/sbin/setenforce permissive 2024-06-18T02:32:11.954 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-06-18T02:32:11.965 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-06-18T02:32:11.968 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-06-18T02:32:12.094 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-06-18T02:32:12.108 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-06-18T02:32:12.115 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi163.front.sepia.ceph.com,smithi186.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-06-18T02:36:31.283 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi163.front.sepia.ceph.com'), Remote(name='ubuntu@smithi186.front.sepia.ceph.com')] 2024-06-18T02:36:31.284 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi163.front.sepia.ceph.com' 2024-06-18T02:36:31.286 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-18T02:36:31.365 DEBUG:teuthology.orchestra.run.smithi163:> true 2024-06-18T02:36:31.453 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi163.front.sepia.ceph.com' 2024-06-18T02:36:31.454 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi186.front.sepia.ceph.com' 2024-06-18T02:36:31.454 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi186.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-18T02:36:31.530 DEBUG:teuthology.orchestra.run.smithi186:> true 2024-06-18T02:36:31.567 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi186.front.sepia.ceph.com' 2024-06-18T02:36:31.567 INFO:teuthology.run_tasks:Running task clock... 2024-06-18T02:36:31.577 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-06-18T02:36:31.578 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-06-18T02:36:31.578 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-18T02:36:31.581 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-06-18T02:36:31.581 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-18T02:36:31.607 INFO:teuthology.orchestra.run.smithi163.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-06-18T02:36:31.621 INFO:teuthology.orchestra.run.smithi163.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-06-18T02:36:31.637 INFO:teuthology.orchestra.run.smithi186.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-06-18T02:36:31.646 INFO:teuthology.orchestra.run.smithi163.stderr:sudo: ntpd: command not found 2024-06-18T02:36:31.651 INFO:teuthology.orchestra.run.smithi186.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-06-18T02:36:31.655 INFO:teuthology.orchestra.run.smithi163.stdout:506 Cannot talk to daemon 2024-06-18T02:36:31.670 INFO:teuthology.orchestra.run.smithi163.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-06-18T02:36:31.675 INFO:teuthology.orchestra.run.smithi186.stderr:sudo: ntpd: command not found 2024-06-18T02:36:31.684 INFO:teuthology.orchestra.run.smithi163.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-06-18T02:36:31.686 INFO:teuthology.orchestra.run.smithi186.stdout:506 Cannot talk to daemon 2024-06-18T02:36:31.700 INFO:teuthology.orchestra.run.smithi186.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-06-18T02:36:31.714 INFO:teuthology.orchestra.run.smithi186.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-06-18T02:36:31.734 INFO:teuthology.orchestra.run.smithi163.stderr:bash: line 1: ntpq: command not found 2024-06-18T02:36:31.738 INFO:teuthology.orchestra.run.smithi163.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-18T02:36:31.738 INFO:teuthology.orchestra.run.smithi163.stdout:=============================================================================== 2024-06-18T02:36:31.738 INFO:teuthology.orchestra.run.smithi163.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-18T02:36:31.739 INFO:teuthology.orchestra.run.smithi163.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-18T02:36:31.739 INFO:teuthology.orchestra.run.smithi163.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-18T02:36:31.739 INFO:teuthology.orchestra.run.smithi163.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-18T02:36:31.766 INFO:teuthology.orchestra.run.smithi186.stderr:bash: line 1: ntpq: command not found 2024-06-18T02:36:31.769 INFO:teuthology.orchestra.run.smithi186.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-18T02:36:31.769 INFO:teuthology.orchestra.run.smithi186.stdout:=============================================================================== 2024-06-18T02:36:31.769 INFO:teuthology.orchestra.run.smithi186.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-18T02:36:31.770 INFO:teuthology.orchestra.run.smithi186.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-18T02:36:31.770 INFO:teuthology.orchestra.run.smithi186.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-18T02:36:31.770 INFO:teuthology.orchestra.run.smithi186.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-18T02:36:31.771 INFO:teuthology.run_tasks:Running task cephadm... 2024-06-18T02:36:31.876 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': '3781e3a592387d16e8020f5db99cdee4198f6d00'} 2024-06-18T02:36:31.876 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-06-18T02:36:31.877 INFO:tasks.cephadm:Cluster fsid is 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:36:31.877 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-06-18T02:36:31.877 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.163', 'mon.c': '[v2:172.21.15.163:3301,v1:172.21.15.163:6790]', 'mon.b': '172.21.15.186'} 2024-06-18T02:36:31.877 INFO:tasks.cephadm:First mon is mon.a on smithi163 2024-06-18T02:36:31.878 INFO:tasks.cephadm:First mgr is y 2024-06-18T02:36:31.878 INFO:tasks.cephadm:Normalizing hostnames... 2024-06-18T02:36:31.878 DEBUG:teuthology.orchestra.run.smithi163:> sudo hostname $(hostname -s) 2024-06-18T02:36:31.904 DEBUG:teuthology.orchestra.run.smithi186:> sudo hostname $(hostname -s) 2024-06-18T02:36:31.931 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-06-18T02:36:31.931 DEBUG:teuthology.orchestra.run.smithi163:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-06-18T02:36:32.173 INFO:teuthology.orchestra.run.smithi163.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Jun 18 02:36 /home/ubuntu/cephtest/cephadm 2024-06-18T02:36:32.173 DEBUG:teuthology.orchestra.run.smithi186:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-06-18T02:36:32.347 INFO:teuthology.orchestra.run.smithi186.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Jun 18 02:36 /home/ubuntu/cephtest/cephadm 2024-06-18T02:36:32.347 DEBUG:teuthology.orchestra.run.smithi163:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-06-18T02:36:32.363 DEBUG:teuthology.orchestra.run.smithi186:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-06-18T02:36:32.389 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-06-18T02:36:32.390 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-06-18T02:36:32.405 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-06-18T02:36:33.299 INFO:teuthology.orchestra.run.smithi163.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-06-18T02:36:33.311 INFO:teuthology.orchestra.run.smithi186.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-06-18T02:36:50.487 INFO:teuthology.orchestra.run.smithi186.stdout:{ 2024-06-18T02:36:50.487 INFO:teuthology.orchestra.run.smithi186.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-06-18T02:36:50.487 INFO:teuthology.orchestra.run.smithi186.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-06-18T02:36:50.487 INFO:teuthology.orchestra.run.smithi186.stdout: "repo_digests": [ 2024-06-18T02:36:50.487 INFO:teuthology.orchestra.run.smithi186.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-06-18T02:36:50.487 INFO:teuthology.orchestra.run.smithi186.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-06-18T02:36:50.488 INFO:teuthology.orchestra.run.smithi186.stdout: ] 2024-06-18T02:36:50.488 INFO:teuthology.orchestra.run.smithi186.stdout:} 2024-06-18T02:36:50.845 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T02:36:50.845 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-06-18T02:36:50.845 INFO:teuthology.orchestra.run.smithi163.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-06-18T02:36:50.845 INFO:teuthology.orchestra.run.smithi163.stdout: "repo_digests": [ 2024-06-18T02:36:50.845 INFO:teuthology.orchestra.run.smithi163.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-06-18T02:36:50.845 INFO:teuthology.orchestra.run.smithi163.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-06-18T02:36:50.846 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2024-06-18T02:36:50.846 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T02:36:50.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo mkdir -p /etc/ceph 2024-06-18T02:36:50.927 DEBUG:teuthology.orchestra.run.smithi186:> sudo mkdir -p /etc/ceph 2024-06-18T02:36:50.958 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod 777 /etc/ceph 2024-06-18T02:36:50.983 DEBUG:teuthology.orchestra.run.smithi186:> sudo chmod 777 /etc/ceph 2024-06-18T02:36:51.025 INFO:tasks.cephadm:Writing seed config... 2024-06-18T02:36:51.027 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-06-18T02:36:51.027 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-06-18T02:36:51.027 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-06-18T02:36:51.027 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-06-18T02:36:51.027 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-06-18T02:36:51.027 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-06-18T02:36:51.027 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-06-18T02:36:51.027 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-06-18T02:36:51.027 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-06-18T02:36:51.028 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2024-06-18T02:36:51.028 DEBUG:teuthology.orchestra.run.smithi163:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-06-18T02:36:51.043 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 929affee-2d1b-11ef-bc9d-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-06-18T02:36:51.043 DEBUG:teuthology.orchestra.run.smithi163:mon.a> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.a.service 2024-06-18T02:36:51.086 DEBUG:teuthology.orchestra.run.smithi163:mgr.y> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.y.service 2024-06-18T02:36:51.128 INFO:tasks.cephadm:Bootstrapping... 2024-06-18T02:36:51.128 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.163 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-06-18T02:36:51.322 INFO:teuthology.orchestra.run.smithi163.stderr:-------------------------------------------------------------------------------- 2024-06-18T02:36:51.323 INFO:teuthology.orchestra.run.smithi163.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '929affee-2d1b-11ef-bc9d-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.163', '--skip-admin-label'] 2024-06-18T02:36:51.345 INFO:teuthology.orchestra.run.smithi163.stderr:/bin/podman: 5.1.0 2024-06-18T02:36:51.348 INFO:teuthology.orchestra.run.smithi163.stderr:Verifying podman|docker is present... 2024-06-18T02:36:51.370 INFO:teuthology.orchestra.run.smithi163.stderr:/bin/podman: 5.1.0 2024-06-18T02:36:51.372 INFO:teuthology.orchestra.run.smithi163.stderr:Verifying lvm2 is present... 2024-06-18T02:36:51.372 INFO:teuthology.orchestra.run.smithi163.stderr:Verifying time synchronization is in place... 2024-06-18T02:36:51.379 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-06-18T02:36:51.385 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2024-06-18T02:36:51.392 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: enabled 2024-06-18T02:36:51.398 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: active 2024-06-18T02:36:51.399 INFO:teuthology.orchestra.run.smithi163.stderr:Unit chronyd.service is enabled and running 2024-06-18T02:36:51.399 INFO:teuthology.orchestra.run.smithi163.stderr:Repeating the final host check... 2024-06-18T02:36:51.420 INFO:teuthology.orchestra.run.smithi163.stderr:/bin/podman: 5.1.0 2024-06-18T02:36:51.422 INFO:teuthology.orchestra.run.smithi163.stderr:podman (/bin/podman) version 5.1.0 is present 2024-06-18T02:36:51.422 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl is present 2024-06-18T02:36:51.422 INFO:teuthology.orchestra.run.smithi163.stderr:lvcreate is present 2024-06-18T02:36:51.429 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-06-18T02:36:51.435 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2024-06-18T02:36:51.441 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: enabled 2024-06-18T02:36:51.448 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: active 2024-06-18T02:36:51.449 INFO:teuthology.orchestra.run.smithi163.stderr:Unit chronyd.service is enabled and running 2024-06-18T02:36:51.449 INFO:teuthology.orchestra.run.smithi163.stderr:Host looks OK 2024-06-18T02:36:51.449 INFO:teuthology.orchestra.run.smithi163.stderr:Cluster fsid: 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:36:51.449 INFO:teuthology.orchestra.run.smithi163.stderr:Acquiring lock 140100245570560 on /run/cephadm/929affee-2d1b-11ef-bc9d-c7b262605968.lock 2024-06-18T02:36:51.449 INFO:teuthology.orchestra.run.smithi163.stderr:Lock 140100245570560 acquired on /run/cephadm/929affee-2d1b-11ef-bc9d-c7b262605968.lock 2024-06-18T02:36:51.450 INFO:teuthology.orchestra.run.smithi163.stderr:Verifying IP 172.21.15.163 port 3300 ... 2024-06-18T02:36:51.450 INFO:teuthology.orchestra.run.smithi163.stderr:Verifying IP 172.21.15.163 port 6789 ... 2024-06-18T02:36:51.450 INFO:teuthology.orchestra.run.smithi163.stderr:Base mon IP is 172.21.15.163, final addrv is [v2:172.21.15.163:3300,v1:172.21.15.163:6789] 2024-06-18T02:36:51.452 INFO:teuthology.orchestra.run.smithi163.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.163 metric 100 2024-06-18T02:36:51.452 INFO:teuthology.orchestra.run.smithi163.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.163 metric 100 2024-06-18T02:36:51.455 INFO:teuthology.orchestra.run.smithi163.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-06-18T02:36:51.455 INFO:teuthology.orchestra.run.smithi163.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-06-18T02:36:51.455 INFO:teuthology.orchestra.run.smithi163.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1796sec hoplimit 64 pref medium 2024-06-18T02:36:51.458 INFO:teuthology.orchestra.run.smithi163.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-06-18T02:36:51.458 INFO:teuthology.orchestra.run.smithi163.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-06-18T02:36:51.458 INFO:teuthology.orchestra.run.smithi163.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-06-18T02:36:51.458 INFO:teuthology.orchestra.run.smithi163.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-06-18T02:36:51.458 INFO:teuthology.orchestra.run.smithi163.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe88:7d5b/64 scope link 2024-06-18T02:36:51.458 INFO:teuthology.orchestra.run.smithi163.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-06-18T02:36:51.459 INFO:teuthology.orchestra.run.smithi163.stderr:Mon IP `172.21.15.163` is in CIDR network `172.21.0.0/20` 2024-06-18T02:36:51.459 INFO:teuthology.orchestra.run.smithi163.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-06-18T02:36:51.459 INFO:teuthology.orchestra.run.smithi163.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-06-18T02:36:51.560 INFO:teuthology.orchestra.run.smithi163.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-06-18T02:36:51.904 INFO:teuthology.orchestra.run.smithi163.stderr:/bin/podman: Getting image source signatures 2024-06-18T02:36:51.904 INFO:teuthology.orchestra.run.smithi163.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-06-18T02:36:51.904 INFO:teuthology.orchestra.run.smithi163.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-06-18T02:36:51.904 INFO:teuthology.orchestra.run.smithi163.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-06-18T02:36:51.904 INFO:teuthology.orchestra.run.smithi163.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-06-18T02:36:51.904 INFO:teuthology.orchestra.run.smithi163.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-06-18T02:36:51.905 INFO:teuthology.orchestra.run.smithi163.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-06-18T02:36:52.065 INFO:teuthology.orchestra.run.smithi163.stderr:/bin/podman: Writing manifest to image destination 2024-06-18T02:36:52.210 INFO:teuthology.orchestra.run.smithi163.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-06-18T02:36:52.616 INFO:teuthology.orchestra.run.smithi163.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-06-18T02:36:53.150 INFO:teuthology.orchestra.run.smithi163.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-06-18T02:36:53.150 INFO:teuthology.orchestra.run.smithi163.stderr:Extracting ceph user uid/gid from container image... 2024-06-18T02:36:53.574 INFO:teuthology.orchestra.run.smithi163.stderr:stat: 167 167 2024-06-18T02:36:54.022 INFO:teuthology.orchestra.run.smithi163.stderr:Creating initial keys... 2024-06-18T02:36:54.521 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-authtool: AQDG8nBmq5MoGxAAYly58QEoQA5onlvgbD8FOQ== 2024-06-18T02:36:55.494 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-authtool: AQDH8nBmT0IOFxAAHqDUTC3OLxrg8OhhE0dNrA== 2024-06-18T02:36:56.515 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph-authtool: AQDI8nBmF91hGxAAXAuzYWJTVTLlV6HLc03gaA== 2024-06-18T02:36:57.073 INFO:teuthology.orchestra.run.smithi163.stderr:Creating initial monmap... 2024-06-18T02:36:57.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-06-18T02:36:57.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-06-18T02:36:57.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:36:57.488 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-06-18T02:36:58.036 INFO:teuthology.orchestra.run.smithi163.stderr:monmaptool for a [v2:172.21.15.163:3300,v1:172.21.15.163:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-06-18T02:36:58.036 INFO:teuthology.orchestra.run.smithi163.stderr:setting min_mon_release = octopus 2024-06-18T02:36:58.036 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/monmaptool: set fsid to 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:36:58.036 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-06-18T02:36:58.037 INFO:teuthology.orchestra.run.smithi163.stderr: 2024-06-18T02:36:58.037 INFO:teuthology.orchestra.run.smithi163.stderr:Creating mon... 2024-06-18T02:36:59.042 INFO:teuthology.orchestra.run.smithi163.stderr:create mon.a on 2024-06-18T02:36:59.272 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-06-18T02:36:59.420 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-929affee-2d1b-11ef-bc9d-c7b262605968.target → /etc/systemd/system/ceph-929affee-2d1b-11ef-bc9d-c7b262605968.target. 2024-06-18T02:36:59.420 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-929affee-2d1b-11ef-bc9d-c7b262605968.target → /etc/systemd/system/ceph-929affee-2d1b-11ef-bc9d-c7b262605968.target. 2024-06-18T02:36:59.707 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Failed to reset failed state of unit ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.a.service: Unit ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.a.service not loaded. 2024-06-18T02:36:59.714 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Created symlink /etc/systemd/system/ceph-929affee-2d1b-11ef-bc9d-c7b262605968.target.wants/ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.a.service → /etc/systemd/system/ceph-929affee-2d1b-11ef-bc9d-c7b262605968@.service. 2024-06-18T02:37:00.385 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:00 smithi163 podman[24733]: 2024-06-18 02:37:00.345564707 +0000 UTC m=+0.358692186 container start 833ebbd8692aee875c535b7bf1f32acd961dab6aa816ec0f605551d6bfddc10a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, io.buildah.version=1.19.8, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , version=8, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream) 2024-06-18T02:37:00.386 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:00 smithi163 ceph-mon[24751]: mkfs 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:37:00.455 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: disabled 2024-06-18T02:37:00.462 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2024-06-18T02:37:00.462 INFO:teuthology.orchestra.run.smithi163.stderr:firewalld.service is not enabled 2024-06-18T02:37:00.462 INFO:teuthology.orchestra.run.smithi163.stderr:Not possible to enable service . firewalld.service is not available 2024-06-18T02:37:00.463 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for mon to start... 2024-06-18T02:37:00.463 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for mon... 2024-06-18T02:37:00.737 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:00 smithi163 ceph-mon[24751]: mon.a is new leader, mons a in quorum (ranks 0) 2024-06-18T02:37:00.737 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:00 smithi163 bash[24733]: 833ebbd8692aee875c535b7bf1f32acd961dab6aa816ec0f605551d6bfddc10a 2024-06-18T02:37:00.737 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:00 smithi163 systemd[1]: Started Ceph mon.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:37:01.027 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: cluster: 2024-06-18T02:37:01.027 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: id: 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:37:01.027 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-06-18T02:37:01.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2024-06-18T02:37:01.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: services: 2024-06-18T02:37:01.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.644608s) 2024-06-18T02:37:01.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mgr: no daemons active 2024-06-18T02:37:01.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-06-18T02:37:01.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2024-06-18T02:37:01.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: data: 2024-06-18T02:37:01.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-06-18T02:37:01.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-06-18T02:37:01.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-06-18T02:37:01.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: pgs: 2024-06-18T02:37:01.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2024-06-18T02:37:01.555 INFO:teuthology.orchestra.run.smithi163.stderr:mon is available 2024-06-18T02:37:01.555 INFO:teuthology.orchestra.run.smithi163.stderr:Assimilating anything we can from ceph.conf... 2024-06-18T02:37:01.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:01 smithi163 ceph-mon[24751]: mon.a is new leader, mons a in quorum (ranks 0) 2024-06-18T02:37:01.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:01 smithi163 ceph-mon[24751]: monmap e1: 1 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0]} 2024-06-18T02:37:01.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:01 smithi163 ceph-mon[24751]: fsmap 2024-06-18T02:37:01.679 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:01 smithi163 ceph-mon[24751]: osdmap e1: 0 total, 0 up, 0 in 2024-06-18T02:37:01.679 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:01 smithi163 ceph-mon[24751]: mgrmap e1: no daemons active 2024-06-18T02:37:01.679 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:01 smithi163 ceph-mon[24751]: from='client.? 172.21.15.163:0/1703039207' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-18T02:37:02.067 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2024-06-18T02:37:02.068 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: [global] 2024-06-18T02:37:02.068 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: fsid = 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:37:02.068 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.163:3300,v1:172.21.15.163:6789] 2024-06-18T02:37:02.068 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-06-18T02:37:02.068 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-06-18T02:37:02.068 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-06-18T02:37:02.068 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-06-18T02:37:02.068 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2024-06-18T02:37:02.068 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: [mgr] 2024-06-18T02:37:02.068 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-06-18T02:37:02.068 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-06-18T02:37:02.068 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2024-06-18T02:37:02.069 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: [osd] 2024-06-18T02:37:02.069 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-06-18T02:37:02.069 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-06-18T02:37:02.642 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:02 smithi163 ceph-mon[24751]: from='client.? 172.21.15.163:0/2310694320' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-06-18T02:37:02.642 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:02 smithi163 ceph-mon[24751]: from='client.? 172.21.15.163:0/2310694320' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-06-18T02:37:02.644 INFO:teuthology.orchestra.run.smithi163.stderr:Generating new minimal ceph.conf... 2024-06-18T02:37:03.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:03 smithi163 ceph-mon[24751]: from='client.? 172.21.15.163:0/881626531' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:37:03.777 INFO:teuthology.orchestra.run.smithi163.stderr:Restarting the monitor... 2024-06-18T02:37:04.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:03 smithi163 systemd[1]: Stopping Ceph mon.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:37:04.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:03 smithi163 bash[24970]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon.a" found: no such container 2024-06-18T02:37:04.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a[24747]: 2024-06-18T02:37:03.918+0000 7fa7b75d8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T02:37:04.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a[24747]: 2024-06-18T02:37:03.918+0000 7fa7b75d8700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-06-18T02:37:04.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:03 smithi163 podman[24982]: 2024-06-18 02:37:03.991236886 +0000 UTC m=+0.154199599 container died 833ebbd8692aee875c535b7bf1f32acd961dab6aa816ec0f605551d6bfddc10a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, name=centos-stream, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, release=754, vcs-type=git, architecture=x86_64, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, com.redhat.component=centos-stream-container) 2024-06-18T02:37:04.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 podman[24982]: 2024-06-18 02:37:04.335881879 +0000 UTC m=+0.498844593 container cleanup 833ebbd8692aee875c535b7bf1f32acd961dab6aa816ec0f605551d6bfddc10a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, release=754, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, ceph=True, vcs-type=git, GIT_CLEAN=True) 2024-06-18T02:37:04.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 bash[24982]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a 2024-06-18T02:37:04.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 podman[24997]: 2024-06-18 02:37:04.520157199 +0000 UTC m=+0.525503638 container remove 833ebbd8692aee875c535b7bf1f32acd961dab6aa816ec0f605551d6bfddc10a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, release=754, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc.) 2024-06-18T02:37:04.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 bash[25009]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon.a" found: no such container 2024-06-18T02:37:04.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.a.service: Deactivated successfully. 2024-06-18T02:37:04.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 systemd[1]: Stopped Ceph mon.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:37:04.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 systemd[1]: Starting Ceph mon.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:37:04.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 podman[25076]: 2024-06-18 02:37:04.754404374 +0000 UTC m=+0.098308541 container create 60c2a10670680cb82ecf9c2b90540ea3d70fe1b12e7de6d6b36ee87c71b5d3f3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, vcs-type=git, io.openshift.expose-services=, architecture=x86_64, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, name=centos-stream, version=8) 2024-06-18T02:37:04.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 podman[25076]: 2024-06-18 02:37:04.673838673 +0000 UTC m=+0.017742853 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-06-18T02:37:05.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 podman[25076]: 2024-06-18 02:37:04.973606714 +0000 UTC m=+0.317510894 container init 60c2a10670680cb82ecf9c2b90540ea3d70fe1b12e7de6d6b36ee87c71b5d3f3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, build-date=2022-05-03T08:36:31.336870, ceph=True, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, distribution-scope=public, name=centos-stream, release=754, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, version=8) 2024-06-18T02:37:05.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 podman[25076]: 2024-06-18 02:37:04.97669434 +0000 UTC m=+0.320598508 container start 60c2a10670680cb82ecf9c2b90540ea3d70fe1b12e7de6d6b36ee87c71b5d3f3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, com.redhat.component=centos-stream-container, architecture=x86_64, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux ) 2024-06-18T02:37:05.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 ceph-mon[25094]: set uid:gid to 167:167 (ceph:ceph) 2024-06-18T02:37:05.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 ceph-mon[25094]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-06-18T02:37:05.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:04 smithi163 ceph-mon[25094]: pidfile_write: ignore empty --pid-file 2024-06-18T02:37:05.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: load: jerasure load: lrc 2024-06-18T02:37:05.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: RocksDB version: 6.15.5 2024-06-18T02:37:05.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-06-18T02:37:05.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Compile date Apr 18 2022 2024-06-18T02:37:05.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: DB SUMMARY 2024-06-18T02:37:05.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: DB Session ID: AFSUWPUSC24ZS83SLPER 2024-06-18T02:37:05.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: CURRENT file: CURRENT 2024-06-18T02:37:05.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: IDENTITY file: IDENTITY 2024-06-18T02:37:05.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-06-18T02:37:05.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-06-18T02:37:05.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82659 ; 2024-06-18T02:37:05.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.error_if_exists: 0 2024-06-18T02:37:05.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.create_if_missing: 0 2024-06-18T02:37:05.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.paranoid_checks: 1 2024-06-18T02:37:05.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-18T02:37:05.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.env: 0x5596332c6860 2024-06-18T02:37:05.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.fs: Posix File System 2024-06-18T02:37:05.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.info_log: 0x559634de7dc0 2024-06-18T02:37:05.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_file_opening_threads: 16 2024-06-18T02:37:05.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.statistics: (nil) 2024-06-18T02:37:05.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.use_fsync: 0 2024-06-18T02:37:05.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_log_file_size: 0 2024-06-18T02:37:05.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-18T02:37:05.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-18T02:37:05.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.keep_log_file_num: 1000 2024-06-18T02:37:05.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.recycle_log_file_num: 0 2024-06-18T02:37:05.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.allow_fallocate: 1 2024-06-18T02:37:05.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.allow_mmap_reads: 0 2024-06-18T02:37:05.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.allow_mmap_writes: 0 2024-06-18T02:37:05.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.use_direct_reads: 0 2024-06-18T02:37:05.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-18T02:37:05.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.create_missing_column_families: 0 2024-06-18T02:37:05.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.db_log_dir: 2024-06-18T02:37:05.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-06-18T02:37:05.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-18T02:37:05.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-18T02:37:05.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-18T02:37:05.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-18T02:37:05.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-18T02:37:05.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-18T02:37:05.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.advise_random_on_open: 1 2024-06-18T02:37:05.037 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.db_write_buffer_size: 0 2024-06-18T02:37:05.037 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.write_buffer_manager: 0x559634ed8240 2024-06-18T02:37:05.037 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-18T02:37:05.037 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-06-18T02:37:05.037 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-18T02:37:05.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-18T02:37:05.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.rate_limiter: (nil) 2024-06-18T02:37:05.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-18T02:37:05.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.wal_recovery_mode: 2 2024-06-18T02:37:05.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.enable_thread_tracking: 0 2024-06-18T02:37:05.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.enable_pipelined_write: 0 2024-06-18T02:37:05.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.unordered_write: 0 2024-06-18T02:37:05.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-18T02:37:05.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-18T02:37:05.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-18T02:37:05.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-18T02:37:05.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.row_cache: None 2024-06-18T02:37:05.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.wal_filter: None 2024-06-18T02:37:05.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-18T02:37:05.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.allow_ingest_behind: 0 2024-06-18T02:37:05.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.preserve_deletes: 0 2024-06-18T02:37:05.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.two_write_queues: 0 2024-06-18T02:37:05.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.manual_wal_flush: 0 2024-06-18T02:37:05.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.atomic_flush: 0 2024-06-18T02:37:05.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-18T02:37:05.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-18T02:37:05.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-18T02:37:05.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.log_readahead_size: 0 2024-06-18T02:37:05.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.best_efforts_recovery: 0 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.allow_data_in_errors: 0 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.db_host_id: __hostname__ 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_background_jobs: 2 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_background_compactions: -1 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_subcompactions: 1 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_total_wal_size: 0 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-18T02:37:05.040 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_open_files: -1 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.bytes_per_sync: 0 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compaction_readahead_size: 0 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_background_flushes: -1 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Compression algorithms supported: 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: kZSTD supported: 0 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: kXpressCompression supported: 0 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: kLZ4HCCompression supported: 1 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: kLZ4Compression supported: 1 2024-06-18T02:37:05.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: kBZip2Compression supported: 0 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: kZlibCompression supported: 1 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: kSnappyCompression supported: 1 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.merge_operator: 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compaction_filter: None 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compaction_filter_factory: None 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.sst_partitioner_factory: None 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559634db3d00) 2024-06-18T02:37:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout: cache_index_and_filter_blocks: 1 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: pin_top_level_index_and_filter: 1 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: index_type: 0 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: data_block_index_type: 0 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: index_shortening: 1 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: hash_index_allow_collision: 1 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: checksum: 1 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: no_block_cache: 0 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache: 0x559634e1e170 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache_name: BinnedLRUCache 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache_options: 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: capacity : 536870912 2024-06-18T02:37:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: num_shard_bits : 4 2024-06-18T02:37:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: strict_capacity_limit : 0 2024-06-18T02:37:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: high_pri_pool_ratio: 0.000 2024-06-18T02:37:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache_compressed: (nil) 2024-06-18T02:37:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: persistent_cache: (nil) 2024-06-18T02:37:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_size: 4096 2024-06-18T02:37:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_size_deviation: 10 2024-06-18T02:37:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_restart_interval: 16 2024-06-18T02:37:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: index_block_restart_interval: 1 2024-06-18T02:37:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: metadata_block_size: 4096 2024-06-18T02:37:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: partition_filters: 0 2024-06-18T02:37:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: use_delta_encoding: 1 2024-06-18T02:37:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-06-18T02:37:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: whole_key_filtering: 1 2024-06-18T02:37:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: verify_compression: 0 2024-06-18T02:37:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: read_amp_bytes_per_bit: 0 2024-06-18T02:37:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: format_version: 4 2024-06-18T02:37:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: enable_index_compression: 1 2024-06-18T02:37:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_align: 0 2024-06-18T02:37:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.write_buffer_size: 33554432 2024-06-18T02:37:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_write_buffer_number: 2 2024-06-18T02:37:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compression: NoCompression 2024-06-18T02:37:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.bottommost_compression: Disabled 2024-06-18T02:37:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.prefix_extractor: nullptr 2024-06-18T02:37:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-18T02:37:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.num_levels: 7 2024-06-18T02:37:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-18T02:37:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-18T02:37:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-18T02:37:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-18T02:37:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-18T02:37:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compression_opts.level: 32767 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compression_opts.strategy: 0 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compression_opts.enabled: false 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-18T02:37:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-18T02:37:05.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.target_file_size_base: 67108864 2024-06-18T02:37:05.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-18T02:37:05.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-18T02:37:05.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-18T02:37:05.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-18T02:37:05.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-18T02:37:05.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-18T02:37:05.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-18T02:37:05.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-18T02:37:05.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-18T02:37:05.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-18T02:37:05.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-18T02:37:05.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.arena_block_size: 4194304 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.disable_auto_compactions: 0 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-18T02:37:05.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.table_properties_collectors: 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.inplace_update_support: 0 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.bloom_locality: 0 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.max_successive_merges: 0 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.paranoid_file_checks: 0 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.force_consistency_checks: 1 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.report_bg_io_stats: 0 2024-06-18T02:37:05.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.ttl: 2592000 2024-06-18T02:37:05.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-18T02:37:05.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.enable_blob_files: false 2024-06-18T02:37:05.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.min_blob_size: 0 2024-06-18T02:37:05.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.blob_file_size: 268435456 2024-06-18T02:37:05.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-18T02:37:05.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-18T02:37:05.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-18T02:37:05.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-18T02:37:05.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-06-18T02:37:05.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-06-18T02:37:05.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718678225002250, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-06-18T02:37:05.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-06-18T02:37:05.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718678225002963, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79209, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77472, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71781, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1718678225, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "b39d3d03-9637-4565-bdeb-ab6a9323dbaa", "db_session_id": "AFSUWPUSC24ZS83SLPER"}} 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718678225003303, "job": 1, "event": "recovery_finished"} 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x559634e04a80 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: DB pointer 0x559634e14000 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** DB Stats ** 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-18T02:37:05.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** Compaction Stats [default] ** 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: L0 2/0 79.11 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.4 0.00 0.00 1 0.000 0 0 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: Sum 2/0 79.11 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.4 0.00 0.00 1 0.000 0 0 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.4 0.00 0.00 1 0.000 0 0 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** Compaction Stats [default] ** 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-18T02:37:05.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 158.4 0.00 0.00 1 0.000 0 0 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative compaction: 0.00 GB write, 25.15 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval compaction: 0.00 GB write, 25.15 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** Compaction Stats [default] ** 2024-06-18T02:37:05.054 INFO:journalctl@ceph.mon.a.smithi163.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-18T02:37:05.055 INFO:journalctl@ceph.mon.a.smithi163.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-18T02:37:05.055 INFO:journalctl@ceph.mon.a.smithi163.stdout: L0 2/0 79.11 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.4 0.00 0.00 1 0.000 0 0 2024-06-18T02:37:05.055 INFO:journalctl@ceph.mon.a.smithi163.stdout: Sum 2/0 79.11 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.4 0.00 0.00 1 0.000 0 0 2024-06-18T02:37:05.055 INFO:journalctl@ceph.mon.a.smithi163.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-06-18T02:37:05.055 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2024-06-18T02:37:05.055 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** Compaction Stats [default] ** 2024-06-18T02:37:05.055 INFO:journalctl@ceph.mon.a.smithi163.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-18T02:37:05.055 INFO:journalctl@ceph.mon.a.smithi163.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-18T02:37:05.055 INFO:journalctl@ceph.mon.a.smithi163.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 158.4 0.00 0.00 1 0.000 0 0 2024-06-18T02:37:05.055 INFO:journalctl@ceph.mon.a.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-18T02:37:05.055 INFO:journalctl@ceph.mon.a.smithi163.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-18T02:37:05.056 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-18T02:37:05.056 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-18T02:37:05.056 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-18T02:37:05.056 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-18T02:37:05.056 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative compaction: 0.00 GB write, 24.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T02:37:05.056 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T02:37:05.056 INFO:journalctl@ceph.mon.a.smithi163.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-18T02:37:05.056 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2024-06-18T02:37:05.056 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-18T02:37:05.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: starting mon.a rank 0 at public addrs [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] at bind addrs [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: mon.a@-1(???) e1 preinit fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: mon.a@-1(???).mds e1 new map 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: mon.a@-1(???).mds e1 print_map 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout: e1 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout: legacy client fscid: -1 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout: No filesystems configured 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-06-18T02:37:05.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-06-18T02:37:05.058 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 bash[25076]: 60c2a10670680cb82ecf9c2b90540ea3d70fe1b12e7de6d6b36ee87c71b5d3f3 2024-06-18T02:37:05.058 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 systemd[1]: Started Ceph mon.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:37:05.058 INFO:teuthology.orchestra.run.smithi163.stderr:Setting mon public_network to 172.21.0.0/20 2024-06-18T02:37:05.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: mon.a is new leader, mons a in quorum (ranks 0) 2024-06-18T02:37:05.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: monmap e1: 1 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0]} 2024-06-18T02:37:05.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: fsmap 2024-06-18T02:37:05.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: osdmap e1: 0 total, 0 up, 0 in 2024-06-18T02:37:05.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:05 smithi163 ceph-mon[25094]: mgrmap e1: no daemons active 2024-06-18T02:37:06.058 INFO:teuthology.orchestra.run.smithi163.stderr:Wrote config to /etc/ceph/ceph.conf 2024-06-18T02:37:06.058 INFO:teuthology.orchestra.run.smithi163.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-06-18T02:37:06.058 INFO:teuthology.orchestra.run.smithi163.stderr:Creating mgr... 2024-06-18T02:37:06.058 INFO:teuthology.orchestra.run.smithi163.stderr:Verifying port 9283 ... 2024-06-18T02:37:06.227 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Failed to reset failed state of unit ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.y.service: Unit ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.y.service not loaded. 2024-06-18T02:37:06.234 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: Created symlink /etc/systemd/system/ceph-929affee-2d1b-11ef-bc9d-c7b262605968.target.wants/ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-929affee-2d1b-11ef-bc9d-c7b262605968@.service. 2024-06-18T02:37:06.660 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:06 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/136576654' entity='client.admin' 2024-06-18T02:37:06.887 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: disabled 2024-06-18T02:37:06.895 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2024-06-18T02:37:06.896 INFO:teuthology.orchestra.run.smithi163.stderr:firewalld.service is not enabled 2024-06-18T02:37:06.896 INFO:teuthology.orchestra.run.smithi163.stderr:Not possible to enable service . firewalld.service is not available 2024-06-18T02:37:06.904 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: disabled 2024-06-18T02:37:06.911 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2024-06-18T02:37:06.912 INFO:teuthology.orchestra.run.smithi163.stderr:firewalld.service is not enabled 2024-06-18T02:37:06.912 INFO:teuthology.orchestra.run.smithi163.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-06-18T02:37:06.912 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for mgr to start... 2024-06-18T02:37:06.912 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for mgr... 2024-06-18T02:37:06.921 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:06 smithi163 systemd[1]: Started Ceph mgr.y for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:37:07.177 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:06 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:06.921+0000 7f5a996d4000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T02:37:07.177 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:07 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:07.060+0000 7f5a996d4000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:37:07.178 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:07 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:07.160+0000 7f5a996d4000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T02:37:07.459 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2024-06-18T02:37:07.460 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2024-06-18T02:37:07.460 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "fsid": "929affee-2d1b-11ef-bc9d-c7b262605968", 2024-06-18T02:37:07.460 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "health": { 2024-06-18T02:37:07.460 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-06-18T02:37:07.460 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "checks": {}, 2024-06-18T02:37:07.461 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mutes": [] 2024-06-18T02:37:07.461 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:07.461 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-06-18T02:37:07.461 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum": [ 2024-06-18T02:37:07.461 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 0 2024-06-18T02:37:07.461 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2024-06-18T02:37:07.462 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum_names": [ 2024-06-18T02:37:07.462 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "a" 2024-06-18T02:37:07.462 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2024-06-18T02:37:07.462 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-06-18T02:37:07.462 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "monmap": { 2024-06-18T02:37:07.462 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-18T02:37:07.462 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-06-18T02:37:07.463 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_mons": 1 2024-06-18T02:37:07.463 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:07.463 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osdmap": { 2024-06-18T02:37:07.463 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-18T02:37:07.463 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_osds": 0, 2024-06-18T02:37:07.463 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-06-18T02:37:07.463 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-06-18T02:37:07.464 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-06-18T02:37:07.470 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-06-18T02:37:07.470 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-06-18T02:37:07.470 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:07.470 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "pgmap": { 2024-06-18T02:37:07.470 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-06-18T02:37:07.470 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-06-18T02:37:07.471 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_pools": 0, 2024-06-18T02:37:07.471 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_objects": 0, 2024-06-18T02:37:07.471 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-06-18T02:37:07.471 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-06-18T02:37:07.471 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-06-18T02:37:07.471 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_total": 0 2024-06-18T02:37:07.471 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:07.472 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "fsmap": { 2024-06-18T02:37:07.472 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-18T02:37:07.472 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "by_rank": [], 2024-06-18T02:37:07.472 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "up:standby": 0 2024-06-18T02:37:07.472 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:07.472 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mgrmap": { 2024-06-18T02:37:07.472 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "available": false, 2024-06-18T02:37:07.473 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-06-18T02:37:07.473 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "modules": [ 2024-06-18T02:37:07.473 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "iostat", 2024-06-18T02:37:07.473 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "nfs", 2024-06-18T02:37:07.473 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "restful" 2024-06-18T02:37:07.473 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2024-06-18T02:37:07.474 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "services": {} 2024-06-18T02:37:07.474 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:07.474 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "servicemap": { 2024-06-18T02:37:07.474 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-18T02:37:07.474 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "modified": "2024-06-18T02:37:00.383192+0000", 2024-06-18T02:37:07.475 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "services": {} 2024-06-18T02:37:07.475 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:07.475 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "progress_events": {} 2024-06-18T02:37:07.475 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2024-06-18T02:37:07.765 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:07 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1820145283' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-06-18T02:37:07.765 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:07 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:07.484+0000 7f5a996d4000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T02:37:07.765 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:07 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:07.569+0000 7f5a996d4000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T02:37:08.013 INFO:teuthology.orchestra.run.smithi163.stderr:mgr not available, waiting (1/15)... 2024-06-18T02:37:09.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:08 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:08.717+0000 7f5a996d4000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T02:37:09.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:08 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:08.880+0000 7f5a996d4000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T02:37:09.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:08 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:08.961+0000 7f5a996d4000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T02:37:09.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:09 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:09.277+0000 7f5a996d4000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T02:37:09.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:09 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:09.384+0000 7f5a996d4000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T02:37:09.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:09 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:09.476+0000 7f5a996d4000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T02:37:10.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:09 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:09.716+0000 7f5a996d4000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:37:10.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:10 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:10.167+0000 7f5a996d4000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T02:37:11.037 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:10 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:10.677+0000 7f5a996d4000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T02:37:11.037 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:10 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:10.770+0000 7f5a996d4000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T02:37:11.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:11.065+0000 7f5a996d4000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T02:37:11.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:11.154+0000 7f5a996d4000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T02:37:11.923 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:11.581+0000 7f5a996d4000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T02:37:11.923 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:11.715+0000 7f5a996d4000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T02:37:11.923 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:11.808+0000 7f5a996d4000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T02:37:11.923 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:11.923+0000 7f5a996d4000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T02:37:12.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:12.012+0000 7f5a996d4000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T02:37:12.915 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:12.556+0000 7f5a996d4000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T02:37:13.193 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:12.915+0000 7f5a996d4000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T02:37:13.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-mon[25094]: Activating manager daemon y 2024-06-18T02:37:13.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-mon[25094]: mgrmap e2: y(active, starting, since 0.00260842s) 2024-06-18T02:37:13.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-mon[25094]: from='mgr.14100 172.21.15.163:0/2053451180' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T02:37:13.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-mon[25094]: from='mgr.14100 172.21.15.163:0/2053451180' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T02:37:13.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-mon[25094]: from='mgr.14100 172.21.15.163:0/2053451180' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T02:37:13.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-mon[25094]: from='mgr.14100 172.21.15.163:0/2053451180' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:37:13.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-mon[25094]: from='mgr.14100 172.21.15.163:0/2053451180' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T02:37:13.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-mon[25094]: Manager daemon y is now available 2024-06-18T02:37:13.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-mon[25094]: from='mgr.14100 172.21.15.163:0/2053451180' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:37:13.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-mon[25094]: from='mgr.14100 172.21.15.163:0/2053451180' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:37:13.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-mon[25094]: from='mgr.14100 172.21.15.163:0/2053451180' entity='mgr.y' 2024-06-18T02:37:13.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-mon[25094]: from='mgr.14100 172.21.15.163:0/2053451180' entity='mgr.y' 2024-06-18T02:37:13.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:12 smithi163 ceph-mon[25094]: from='mgr.14100 172.21.15.163:0/2053451180' entity='mgr.y' 2024-06-18T02:37:14.002 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2024-06-18T02:37:14.002 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2024-06-18T02:37:14.002 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "fsid": "929affee-2d1b-11ef-bc9d-c7b262605968", 2024-06-18T02:37:14.002 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "health": { 2024-06-18T02:37:14.002 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-06-18T02:37:14.002 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "checks": {}, 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mutes": [] 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum": [ 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 0 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum_names": [ 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "a" 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "monmap": { 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-06-18T02:37:14.003 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_mons": 1 2024-06-18T02:37:14.004 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:14.004 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osdmap": { 2024-06-18T02:37:14.004 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-18T02:37:14.004 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_osds": 0, 2024-06-18T02:37:14.004 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-06-18T02:37:14.004 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-06-18T02:37:14.004 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-06-18T02:37:14.007 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-06-18T02:37:14.007 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-06-18T02:37:14.007 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:14.007 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "pgmap": { 2024-06-18T02:37:14.007 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_pools": 0, 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_objects": 0, 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_total": 0 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "fsmap": { 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "by_rank": [], 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "up:standby": 0 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:14.008 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mgrmap": { 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "available": false, 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "modules": [ 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "iostat", 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "nfs", 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "restful" 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "services": {} 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "servicemap": { 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "modified": "2024-06-18T02:37:00.383192+0000", 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "services": {} 2024-06-18T02:37:14.009 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:14.010 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "progress_events": {} 2024-06-18T02:37:14.010 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2024-06-18T02:37:14.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:13 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/79532512' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-06-18T02:37:14.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:13 smithi163 ceph-mon[25094]: mgrmap e3: y(active, since 1.00577s) 2024-06-18T02:37:16.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:15 smithi163 ceph-mon[25094]: mgrmap e4: y(active, since 2s) 2024-06-18T02:37:19.159 INFO:teuthology.orchestra.run.smithi163.stderr:mgr not available, waiting (2/15)... 2024-06-18T02:37:21.848 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2024-06-18T02:37:21.849 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2024-06-18T02:37:21.849 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "fsid": "929affee-2d1b-11ef-bc9d-c7b262605968", 2024-06-18T02:37:21.849 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "health": { 2024-06-18T02:37:21.849 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-06-18T02:37:21.849 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "checks": {}, 2024-06-18T02:37:21.850 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mutes": [] 2024-06-18T02:37:21.850 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:21.850 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-06-18T02:37:21.850 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum": [ 2024-06-18T02:37:21.850 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 0 2024-06-18T02:37:21.851 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2024-06-18T02:37:21.851 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum_names": [ 2024-06-18T02:37:21.851 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "a" 2024-06-18T02:37:21.851 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2024-06-18T02:37:21.851 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "quorum_age": 16, 2024-06-18T02:37:21.851 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "monmap": { 2024-06-18T02:37:21.852 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-18T02:37:21.852 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-06-18T02:37:21.852 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_mons": 1 2024-06-18T02:37:21.852 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:21.852 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osdmap": { 2024-06-18T02:37:21.852 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-18T02:37:21.852 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_osds": 0, 2024-06-18T02:37:21.852 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-06-18T02:37:21.853 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-06-18T02:37:21.859 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-06-18T02:37:21.859 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-06-18T02:37:21.859 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-06-18T02:37:21.860 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:21.860 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "pgmap": { 2024-06-18T02:37:21.860 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-06-18T02:37:21.860 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-06-18T02:37:21.860 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_pools": 0, 2024-06-18T02:37:21.860 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_objects": 0, 2024-06-18T02:37:21.861 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-06-18T02:37:21.861 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-06-18T02:37:21.861 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-06-18T02:37:21.861 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "bytes_total": 0 2024-06-18T02:37:21.861 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:21.861 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "fsmap": { 2024-06-18T02:37:21.861 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-18T02:37:21.862 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "by_rank": [], 2024-06-18T02:37:21.862 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "up:standby": 0 2024-06-18T02:37:21.862 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:21.862 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mgrmap": { 2024-06-18T02:37:21.862 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "available": true, 2024-06-18T02:37:21.862 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-06-18T02:37:21.862 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "modules": [ 2024-06-18T02:37:21.863 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "iostat", 2024-06-18T02:37:21.863 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "nfs", 2024-06-18T02:37:21.863 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "restful" 2024-06-18T02:37:21.863 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ], 2024-06-18T02:37:21.863 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "services": {} 2024-06-18T02:37:21.863 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:21.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "servicemap": { 2024-06-18T02:37:21.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-18T02:37:21.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "modified": "2024-06-18T02:37:00.383192+0000", 2024-06-18T02:37:21.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "services": {} 2024-06-18T02:37:21.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: }, 2024-06-18T02:37:21.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "progress_events": {} 2024-06-18T02:37:21.864 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2024-06-18T02:37:22.196 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:21 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1535528282' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-06-18T02:37:22.449 INFO:teuthology.orchestra.run.smithi163.stderr:mgr is available 2024-06-18T02:37:23.027 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2024-06-18T02:37:23.027 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: [global] 2024-06-18T02:37:23.027 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: fsid = 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:37:23.027 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-06-18T02:37:23.027 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-06-18T02:37:23.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-06-18T02:37:23.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-06-18T02:37:23.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2024-06-18T02:37:23.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: [mgr] 2024-06-18T02:37:23.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-06-18T02:37:23.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 2024-06-18T02:37:23.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: [osd] 2024-06-18T02:37:23.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-06-18T02:37:23.028 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-06-18T02:37:23.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:23 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2042746319' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-06-18T02:37:23.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:23 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2042746319' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-06-18T02:37:24.016 INFO:teuthology.orchestra.run.smithi163.stderr:Enabling cephadm module... 2024-06-18T02:37:26.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:26 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1057366878' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-06-18T02:37:26.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ignoring --setuser ceph since I am not root 2024-06-18T02:37:26.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ignoring --setgroup ceph since I am not root 2024-06-18T02:37:26.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:26.180+0000 7fdc842fd000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T02:37:26.616 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:26.312+0000 7fdc842fd000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:37:26.616 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:26.411+0000 7fdc842fd000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T02:37:26.944 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:26.728+0000 7fdc842fd000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T02:37:26.944 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:26.812+0000 7fdc842fd000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T02:37:27.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:27 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1057366878' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-06-18T02:37:27.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:27 smithi163 ceph-mon[25094]: mgrmap e5: y(active, since 13s) 2024-06-18T02:37:27.312 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2024-06-18T02:37:27.312 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 5, 2024-06-18T02:37:27.312 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "available": true, 2024-06-18T02:37:27.312 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "active_name": "y", 2024-06-18T02:37:27.312 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_standby": 0 2024-06-18T02:37:27.312 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2024-06-18T02:37:27.902 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for the mgr to restart... 2024-06-18T02:37:27.902 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for mgr epoch 5... 2024-06-18T02:37:28.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:28 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3765031431' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-06-18T02:37:28.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:27.976+0000 7fdc842fd000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T02:37:28.220 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:28.145+0000 7fdc842fd000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T02:37:28.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:28.227+0000 7fdc842fd000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T02:37:28.970 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:28.548+0000 7fdc842fd000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T02:37:28.970 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:28.638+0000 7fdc842fd000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T02:37:28.970 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:28.730+0000 7fdc842fd000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T02:37:28.970 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:28.970+0000 7fdc842fd000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:37:29.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:29.403+0000 7fdc842fd000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T02:37:30.266 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:29.884+0000 7fdc842fd000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T02:37:30.266 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:29.976+0000 7fdc842fd000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T02:37:30.266 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:30.266+0000 7fdc842fd000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T02:37:30.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:30.353+0000 7fdc842fd000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T02:37:31.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:30.756+0000 7fdc842fd000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T02:37:31.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:30.886+0000 7fdc842fd000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T02:37:31.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:30.979+0000 7fdc842fd000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T02:37:31.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:31 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:31.093+0000 7fdc842fd000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T02:37:31.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:31 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:31.180+0000 7fdc842fd000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T02:37:32.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:31 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:31.719+0000 7fdc842fd000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T02:37:32.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:32 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:32.084+0000 7fdc842fd000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T02:37:32.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:32 smithi163 ceph-mon[25094]: Active manager daemon y restarted 2024-06-18T02:37:32.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:32 smithi163 ceph-mon[25094]: Activating manager daemon y 2024-06-18T02:37:32.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:32 smithi163 ceph-mon[25094]: osdmap e2: 0 total, 0 up, 0 in 2024-06-18T02:37:33.150 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2024-06-18T02:37:33.150 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-06-18T02:37:33.150 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "initialized": true 2024-06-18T02:37:33.150 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2024-06-18T02:37:33.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: mgrmap e6: y(active, starting, since 0.0527127s) 2024-06-18T02:37:33.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:37:33.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T02:37:33.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T02:37:33.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T02:37:33.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T02:37:33.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: Manager daemon y is now available 2024-06-18T02:37:33.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' 2024-06-18T02:37:33.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' 2024-06-18T02:37:33.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:37:33.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:37:33.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:37:33.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:37:33.449 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:33 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:37:33.717 INFO:teuthology.orchestra.run.smithi163.stderr:mgr epoch 5 is available 2024-06-18T02:37:33.717 INFO:teuthology.orchestra.run.smithi163.stderr:Setting orchestrator backend to cephadm... 2024-06-18T02:37:34.358 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:34 smithi163 ceph-mon[25094]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-06-18T02:37:34.358 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:34 smithi163 ceph-mon[25094]: mgrmap e7: y(active, since 1.05797s) 2024-06-18T02:37:34.359 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:34 smithi163 ceph-mon[25094]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-06-18T02:37:35.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:35 smithi163 ceph-mon[25094]: mgrmap e8: y(active, since 2s) 2024-06-18T02:37:35.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:35 smithi163 ceph-mon[25094]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:37:35.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:35 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' 2024-06-18T02:37:35.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:35 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:37:35.410 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: value unchanged 2024-06-18T02:37:35.764 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:35 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:37:35] ENGINE Bus STARTING 2024-06-18T02:37:35.764 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:35 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:37:35] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T02:37:35.764 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:35 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:37:35] ENGINE Bus STARTED 2024-06-18T02:37:36.071 INFO:teuthology.orchestra.run.smithi163.stderr:Generating ssh key... 2024-06-18T02:37:36.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-mon[25094]: [18/Jun/2024:02:37:35] ENGINE Bus STARTING 2024-06-18T02:37:36.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-mon[25094]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:37:36.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-mon[25094]: [18/Jun/2024:02:37:35] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T02:37:36.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-mon[25094]: [18/Jun/2024:02:37:35] ENGINE Bus STARTED 2024-06-18T02:37:36.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' 2024-06-18T02:37:36.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:37:37.199 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: Generating public/private rsa key pair. 2024-06-18T02:37:37.199 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: Your identification has been saved in /tmp/tmpdr4pcjyv/key. 2024-06-18T02:37:37.199 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: Your public key has been saved in /tmp/tmpdr4pcjyv/key.pub. 2024-06-18T02:37:37.199 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: The key fingerprint is: 2024-06-18T02:37:37.199 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: SHA256:8fBtS7iWw+NliCk94tVxEoYmPRJqjqSazQGHyNyuwDk ceph-929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:37:37.200 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: The key's randomart image is: 2024-06-18T02:37:37.200 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: +---[RSA 3072]----+ 2024-06-18T02:37:37.200 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: | . | 2024-06-18T02:37:37.200 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: |+.. . o . | 2024-06-18T02:37:37.200 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: |+oo.o o B o | 2024-06-18T02:37:37.201 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: |.=o+ + B + | 2024-06-18T02:37:37.201 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: |oEo.. S * = | 2024-06-18T02:37:37.201 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: |o+o. . = X . | 2024-06-18T02:37:37.201 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: |o.o o * X + | 2024-06-18T02:37:37.201 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: | . + + = | 2024-06-18T02:37:37.201 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: | . . | 2024-06-18T02:37:37.202 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:36 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: +----[SHA256]-----+ 2024-06-18T02:37:38.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:37 smithi163 ceph-mon[25094]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:37:38.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:37 smithi163 ceph-mon[25094]: Generating ssh key... 2024-06-18T02:37:38.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:37 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' 2024-06-18T02:37:38.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:37 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' 2024-06-18T02:37:38.015 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6hNqWp4WNGnCPyCZKaAVhwjs5F8A77CptTF6LeeUPVjl2g+udVt4qn/riQajmlhDxcQlokW4Cdy8ZTTFsh9i3p9VSe1Wj9yOuLk2A4d9aUVWty/8JZLBwrSm7eNDbBnrx+j/PLcYuQZfEOrbYGrKKVxu17TgYnRKk/s/2+Fl4st6E4dQA0lUJMR6H1kJrzITJ4dudMzH4zBtBaKt+bICgNdaLFqyXqBUtQuaYLOIoNtcuIyvxguaMjIIo5nN5LG0poFX21Rb0ZO+fxT4gNA1rimFP4R3ZaA7SAvQvRiAVzGJzj9KQW4SNXzmOQPcYsqPkbRl8xNDGfpwpTu7OcbpEJRvOcKSmSBiuQZbAWVQ7tDqr2dqJrRtRrmvyUEN7x5EIaAb2hpjDTxGxmqCe9K1clmRPi27MWj0wKVdJKsu5L6giHAxds09Ux/WK+1wKQEnJc8NSJppvJ0nT+uKc+SvjZnSQqoMYdwlWArRgU04pmttHzxen79PY+eGRLXGhBbk= ceph-929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:37:38.590 INFO:teuthology.orchestra.run.smithi163.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-06-18T02:37:38.590 INFO:teuthology.orchestra.run.smithi163.stderr:Adding key to root@localhost authorized_keys... 2024-06-18T02:37:38.590 INFO:teuthology.orchestra.run.smithi163.stderr:Adding host smithi163... 2024-06-18T02:37:39.196 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:38 smithi163 ceph-mon[25094]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:37:40.039 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: Added host 'smithi163' with addr '172.21.15.163' 2024-06-18T02:37:40.640 INFO:teuthology.orchestra.run.smithi163.stderr:Deploying unmanaged mon service... 2024-06-18T02:37:41.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:41 smithi163 ceph-mon[25094]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi163", "addr": "172.21.15.163", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:37:41.269 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:41 smithi163 ceph-mon[25094]: Deploying cephadm binary to smithi163 2024-06-18T02:37:41.269 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:41 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' 2024-06-18T02:37:41.269 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:41 smithi163 ceph-mon[25094]: Added host smithi163 2024-06-18T02:37:41.269 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:41 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:37:41.269 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: Scheduled mon update... 2024-06-18T02:37:41.858 INFO:teuthology.orchestra.run.smithi163.stderr:Deploying unmanaged mgr service... 2024-06-18T02:37:42.449 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:42 smithi163 ceph-mon[25094]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:37:42.449 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:42 smithi163 ceph-mon[25094]: Saving service mon spec with placement count:5 2024-06-18T02:37:42.449 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:42 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' 2024-06-18T02:37:42.482 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-06-18T02:37:43.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:43 smithi163 ceph-mon[25094]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:37:43.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:43 smithi163 ceph-mon[25094]: Saving service mgr spec with placement count:2 2024-06-18T02:37:43.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:43 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' 2024-06-18T02:37:44.964 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:44 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' 2024-06-18T02:37:44.964 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:44 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' 2024-06-18T02:37:45.957 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:45 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/580399342' entity='client.admin' 2024-06-18T02:37:45.957 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:45 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' 2024-06-18T02:37:46.816 INFO:teuthology.orchestra.run.smithi163.stderr:Enabling the dashboard module... 2024-06-18T02:37:47.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:47 smithi163 ceph-mon[25094]: from='mgr.14118 172.21.15.163:0/3959453056' entity='mgr.y' 2024-06-18T02:37:47.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:47 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1933069317' entity='client.admin' 2024-06-18T02:37:48.330 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:48 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1201732256' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-06-18T02:37:48.330 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ignoring --setuser ceph since I am not root 2024-06-18T02:37:48.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ignoring --setgroup ceph since I am not root 2024-06-18T02:37:48.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:48.329+0000 7fbea4d74000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T02:37:48.778 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:48.459+0000 7fbea4d74000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:37:48.779 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:48.559+0000 7fbea4d74000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T02:37:49.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:48.868+0000 7fbea4d74000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T02:37:49.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:48.950+0000 7fbea4d74000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T02:37:49.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:49 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1201732256' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-06-18T02:37:49.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:49 smithi163 ceph-mon[25094]: mgrmap e9: y(active, since 16s) 2024-06-18T02:37:49.859 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2024-06-18T02:37:49.859 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "epoch": 9, 2024-06-18T02:37:49.859 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "available": true, 2024-06-18T02:37:49.859 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "active_name": "y", 2024-06-18T02:37:49.859 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "num_standby": 0 2024-06-18T02:37:49.859 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2024-06-18T02:37:50.161 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:50 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:50.099+0000 7fbea4d74000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T02:37:50.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:50 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1954118585' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-06-18T02:37:50.475 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:50 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:50.265+0000 7fbea4d74000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T02:37:50.475 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:50 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:50.346+0000 7fbea4d74000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T02:37:50.477 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for the mgr to restart... 2024-06-18T02:37:50.477 INFO:teuthology.orchestra.run.smithi163.stderr:Waiting for mgr epoch 9... 2024-06-18T02:37:50.767 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:50 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:50.672+0000 7fbea4d74000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T02:37:50.767 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:50 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:50.766+0000 7fbea4d74000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T02:37:51.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:50 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:50.863+0000 7fbea4d74000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T02:37:51.466 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:51 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:51.112+0000 7fbea4d74000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:37:51.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:51 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:51.570+0000 7fbea4d74000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T02:37:52.474 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:52 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:52.075+0000 7fbea4d74000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T02:37:52.475 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:52 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:52.171+0000 7fbea4d74000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T02:37:52.475 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:52 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:52.473+0000 7fbea4d74000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T02:37:52.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:52 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:52.563+0000 7fbea4d74000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T02:37:53.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:52 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:52.971+0000 7fbea4d74000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T02:37:53.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:53.102+0000 7fbea4d74000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T02:37:53.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:53.195+0000 7fbea4d74000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T02:37:53.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:53.308+0000 7fbea4d74000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T02:37:53.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:53.396+0000 7fbea4d74000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T02:37:54.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:53.936+0000 7fbea4d74000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T02:37:54.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:54 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:37:54.309+0000 7fbea4d74000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T02:37:55.160 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:54 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:37:54] ENGINE Bus STARTING 2024-06-18T02:37:55.160 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:54 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:37:54] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T02:37:55.160 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:37:54 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:37:54] ENGINE Bus STARTED 2024-06-18T02:37:55.374 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: { 2024-06-18T02:37:55.375 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-06-18T02:37:55.375 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: "initialized": true 2024-06-18T02:37:55.375 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: } 2024-06-18T02:37:55.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: Active manager daemon y restarted 2024-06-18T02:37:55.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: Activating manager daemon y 2024-06-18T02:37:55.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: osdmap e3: 0 total, 0 up, 0 in 2024-06-18T02:37:55.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: mgrmap e10: y(active, starting, since 0.0556332s) 2024-06-18T02:37:55.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:37:55.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T02:37:55.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T02:37:55.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T02:37:55.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T02:37:55.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: Manager daemon y is now available 2024-06-18T02:37:55.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:37:55.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:37:55.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:37:55.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:37:55.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: [18/Jun/2024:02:37:54] ENGINE Bus STARTING 2024-06-18T02:37:55.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: [18/Jun/2024:02:37:54] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T02:37:55.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: [18/Jun/2024:02:37:54] ENGINE Bus STARTED 2024-06-18T02:37:55.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:37:56.039 INFO:teuthology.orchestra.run.smithi163.stderr:mgr epoch 9 is available 2024-06-18T02:37:56.039 INFO:teuthology.orchestra.run.smithi163.stderr:Generating a dashboard self-signed certificate... 2024-06-18T02:37:56.712 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:56 smithi163 ceph-mon[25094]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-06-18T02:37:56.713 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:56 smithi163 ceph-mon[25094]: mgrmap e11: y(active, since 1.0578s) 2024-06-18T02:37:56.713 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:56 smithi163 ceph-mon[25094]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-06-18T02:37:56.713 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: Self-signed certificate created 2024-06-18T02:37:57.281 INFO:teuthology.orchestra.run.smithi163.stderr:Creating initial admin user... 2024-06-18T02:37:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:57 smithi163 ceph-mon[25094]: mgrmap e12: y(active, since 2s) 2024-06-18T02:37:57.530 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:37:57.530 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:57 smithi163 ceph-mon[25094]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:37:57.530 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:37:57.530 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:37:57.530 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:37:58.174 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$azI/.rWBMORjy3repew7wuDQZvUGOdaDkzOBXAZOD9WukIMflebAm", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1718678278, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-06-18T02:37:59.304 INFO:teuthology.orchestra.run.smithi163.stderr:Fetching dashboard port number... 2024-06-18T02:37:59.433 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:59 smithi163 ceph-mon[25094]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:37:59.433 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:37:59 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:37:59.973 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: 8443 2024-06-18T02:38:00.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:00 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2699769501' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-06-18T02:38:00.520 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: disabled 2024-06-18T02:38:00.527 INFO:teuthology.orchestra.run.smithi163.stderr:systemctl: inactive 2024-06-18T02:38:00.528 INFO:teuthology.orchestra.run.smithi163.stderr:firewalld.service is not enabled 2024-06-18T02:38:00.528 INFO:teuthology.orchestra.run.smithi163.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-06-18T02:38:00.530 INFO:teuthology.orchestra.run.smithi163.stderr:Ceph Dashboard is now available at: 2024-06-18T02:38:00.530 INFO:teuthology.orchestra.run.smithi163.stderr: 2024-06-18T02:38:00.530 INFO:teuthology.orchestra.run.smithi163.stderr: URL: https://smithi163.front.sepia.ceph.com:8443/ 2024-06-18T02:38:00.530 INFO:teuthology.orchestra.run.smithi163.stderr: User: admin 2024-06-18T02:38:00.530 INFO:teuthology.orchestra.run.smithi163.stderr: Password: 5lf37qgppx 2024-06-18T02:38:00.531 INFO:teuthology.orchestra.run.smithi163.stderr: 2024-06-18T02:38:00.531 INFO:teuthology.orchestra.run.smithi163.stderr:Enabling autotune for osd_memory_target 2024-06-18T02:38:02.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:02 smithi163 ceph-mon[25094]: mgrmap e13: y(active, since 6s) 2024-06-18T02:38:02.449 INFO:teuthology.orchestra.run.smithi163.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-06-18T02:38:03.053 INFO:teuthology.orchestra.run.smithi163.stderr:You can access the Ceph CLI with: 2024-06-18T02:38:03.053 INFO:teuthology.orchestra.run.smithi163.stderr: 2024-06-18T02:38:03.053 INFO:teuthology.orchestra.run.smithi163.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-06-18T02:38:03.053 INFO:teuthology.orchestra.run.smithi163.stderr: 2024-06-18T02:38:03.053 INFO:teuthology.orchestra.run.smithi163.stderr:Please consider enabling telemetry to help improve Ceph: 2024-06-18T02:38:03.054 INFO:teuthology.orchestra.run.smithi163.stderr: 2024-06-18T02:38:03.054 INFO:teuthology.orchestra.run.smithi163.stderr: ceph telemetry on 2024-06-18T02:38:03.054 INFO:teuthology.orchestra.run.smithi163.stderr: 2024-06-18T02:38:03.054 INFO:teuthology.orchestra.run.smithi163.stderr:For more information see: 2024-06-18T02:38:03.054 INFO:teuthology.orchestra.run.smithi163.stderr: 2024-06-18T02:38:03.054 INFO:teuthology.orchestra.run.smithi163.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-06-18T02:38:03.054 INFO:teuthology.orchestra.run.smithi163.stderr: 2024-06-18T02:38:03.054 INFO:teuthology.orchestra.run.smithi163.stderr:Bootstrap complete. 2024-06-18T02:38:03.092 INFO:tasks.cephadm:Fetching config... 2024-06-18T02:38:03.093 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2024-06-18T02:38:03.093 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-06-18T02:38:03.109 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-06-18T02:38:03.109 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2024-06-18T02:38:03.110 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-06-18T02:38:03.165 INFO:tasks.cephadm:Fetching mon keyring... 2024-06-18T02:38:03.165 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2024-06-18T02:38:03.165 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/keyring of=/dev/stdout 2024-06-18T02:38:03.231 INFO:tasks.cephadm:Fetching pub ssh key... 2024-06-18T02:38:03.231 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2024-06-18T02:38:03.231 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-06-18T02:38:03.287 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-06-18T02:38:03.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6hNqWp4WNGnCPyCZKaAVhwjs5F8A77CptTF6LeeUPVjl2g+udVt4qn/riQajmlhDxcQlokW4Cdy8ZTTFsh9i3p9VSe1Wj9yOuLk2A4d9aUVWty/8JZLBwrSm7eNDbBnrx+j/PLcYuQZfEOrbYGrKKVxu17TgYnRKk/s/2+Fl4st6E4dQA0lUJMR6H1kJrzITJ4dudMzH4zBtBaKt+bICgNdaLFqyXqBUtQuaYLOIoNtcuIyvxguaMjIIo5nN5LG0poFX21Rb0ZO+fxT4gNA1rimFP4R3ZaA7SAvQvRiAVzGJzj9KQW4SNXzmOQPcYsqPkbRl8xNDGfpwpTu7OcbpEJRvOcKSmSBiuQZbAWVQ7tDqr2dqJrRtRrmvyUEN7x5EIaAb2hpjDTxGxmqCe9K1clmRPi27MWj0wKVdJKsu5L6giHAxds09Ux/WK+1wKQEnJc8NSJppvJ0nT+uKc+SvjZnSQqoMYdwlWArRgU04pmttHzxen79PY+eGRLXGhBbk= ceph-929affee-2d1b-11ef-bc9d-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-06-18T02:38:03.371 INFO:teuthology.orchestra.run.smithi163.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6hNqWp4WNGnCPyCZKaAVhwjs5F8A77CptTF6LeeUPVjl2g+udVt4qn/riQajmlhDxcQlokW4Cdy8ZTTFsh9i3p9VSe1Wj9yOuLk2A4d9aUVWty/8JZLBwrSm7eNDbBnrx+j/PLcYuQZfEOrbYGrKKVxu17TgYnRKk/s/2+Fl4st6E4dQA0lUJMR6H1kJrzITJ4dudMzH4zBtBaKt+bICgNdaLFqyXqBUtQuaYLOIoNtcuIyvxguaMjIIo5nN5LG0poFX21Rb0ZO+fxT4gNA1rimFP4R3ZaA7SAvQvRiAVzGJzj9KQW4SNXzmOQPcYsqPkbRl8xNDGfpwpTu7OcbpEJRvOcKSmSBiuQZbAWVQ7tDqr2dqJrRtRrmvyUEN7x5EIaAb2hpjDTxGxmqCe9K1clmRPi27MWj0wKVdJKsu5L6giHAxds09Ux/WK+1wKQEnJc8NSJppvJ0nT+uKc+SvjZnSQqoMYdwlWArRgU04pmttHzxen79PY+eGRLXGhBbk= ceph-929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:38:03.382 DEBUG:teuthology.orchestra.run.smithi186:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6hNqWp4WNGnCPyCZKaAVhwjs5F8A77CptTF6LeeUPVjl2g+udVt4qn/riQajmlhDxcQlokW4Cdy8ZTTFsh9i3p9VSe1Wj9yOuLk2A4d9aUVWty/8JZLBwrSm7eNDbBnrx+j/PLcYuQZfEOrbYGrKKVxu17TgYnRKk/s/2+Fl4st6E4dQA0lUJMR6H1kJrzITJ4dudMzH4zBtBaKt+bICgNdaLFqyXqBUtQuaYLOIoNtcuIyvxguaMjIIo5nN5LG0poFX21Rb0ZO+fxT4gNA1rimFP4R3ZaA7SAvQvRiAVzGJzj9KQW4SNXzmOQPcYsqPkbRl8xNDGfpwpTu7OcbpEJRvOcKSmSBiuQZbAWVQ7tDqr2dqJrRtRrmvyUEN7x5EIaAb2hpjDTxGxmqCe9K1clmRPi27MWj0wKVdJKsu5L6giHAxds09Ux/WK+1wKQEnJc8NSJppvJ0nT+uKc+SvjZnSQqoMYdwlWArRgU04pmttHzxen79PY+eGRLXGhBbk= ceph-929affee-2d1b-11ef-bc9d-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-06-18T02:38:03.424 INFO:teuthology.orchestra.run.smithi186.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6hNqWp4WNGnCPyCZKaAVhwjs5F8A77CptTF6LeeUPVjl2g+udVt4qn/riQajmlhDxcQlokW4Cdy8ZTTFsh9i3p9VSe1Wj9yOuLk2A4d9aUVWty/8JZLBwrSm7eNDbBnrx+j/PLcYuQZfEOrbYGrKKVxu17TgYnRKk/s/2+Fl4st6E4dQA0lUJMR6H1kJrzITJ4dudMzH4zBtBaKt+bICgNdaLFqyXqBUtQuaYLOIoNtcuIyvxguaMjIIo5nN5LG0poFX21Rb0ZO+fxT4gNA1rimFP4R3ZaA7SAvQvRiAVzGJzj9KQW4SNXzmOQPcYsqPkbRl8xNDGfpwpTu7OcbpEJRvOcKSmSBiuQZbAWVQ7tDqr2dqJrRtRrmvyUEN7x5EIaAb2hpjDTxGxmqCe9K1clmRPi27MWj0wKVdJKsu5L6giHAxds09Ux/WK+1wKQEnJc8NSJppvJ0nT+uKc+SvjZnSQqoMYdwlWArRgU04pmttHzxen79PY+eGRLXGhBbk= ceph-929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:38:03.436 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-06-18T02:38:03.744 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:03 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/215381594' entity='client.admin' 2024-06-18T02:38:06.550 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-06-18T02:38:06.551 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-06-18T02:38:07.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:07 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2543234904' entity='client.admin' 2024-06-18T02:38:09.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:09 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:09.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:09 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi163", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:38:09.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:09 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:09.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:09 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:09.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:09 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:09 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:09.662 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi186 2024-06-18T02:38:09.663 DEBUG:teuthology.orchestra.run.smithi186:> set -ex 2024-06-18T02:38:09.663 DEBUG:teuthology.orchestra.run.smithi186:> dd of=/etc/ceph/ceph.conf 2024-06-18T02:38:09.681 DEBUG:teuthology.orchestra.run.smithi186:> set -ex 2024-06-18T02:38:09.681 DEBUG:teuthology.orchestra.run.smithi186:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:38:09.740 INFO:tasks.cephadm:Adding host smithi186 to orchestrator... 2024-06-18T02:38:09.740 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch host add smithi186 2024-06-18T02:38:10.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:10 smithi163 ceph-mon[25094]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:38:10.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:10 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:10.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:10 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:10.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:10 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:10.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:10 smithi163 ceph-mon[25094]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T02:38:10.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:10 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:10.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:10 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:11.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:11 smithi163 ceph-mon[25094]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:38:12.470 INFO:teuthology.orchestra.run.smithi163.stdout:Added host 'smithi186' with addr '172.21.15.186' 2024-06-18T02:38:12.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:12 smithi163 ceph-mon[25094]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi186", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:38:12.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:12 smithi163 ceph-mon[25094]: Deploying cephadm binary to smithi186 2024-06-18T02:38:13.076 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch host ls --format=json 2024-06-18T02:38:13.674 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:13 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:13.674 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:13 smithi163 ceph-mon[25094]: Added host smithi186 2024-06-18T02:38:13.674 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:13 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:13.674 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:13 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:13.674 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:13 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:13.674 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:13 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:13.674 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:13 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:14.896 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:38:14.897 INFO:teuthology.orchestra.run.smithi163.stdout:[{"addr": "172.21.15.163", "hostname": "smithi163", "labels": [], "status": ""}, {"addr": "172.21.15.186", "hostname": "smithi186", "labels": [], "status": ""}] 2024-06-18T02:38:15.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:15 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:15.443 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:15 smithi163 ceph-mon[25094]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:15.443 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:15 smithi163 ceph-mon[25094]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-18T02:38:15.444 INFO:tasks.cephadm:Setting crush tunables to default 2024-06-18T02:38:15.444 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd crush tunables default 2024-06-18T02:38:18.096 INFO:teuthology.orchestra.run.smithi163.stderr:adjusted tunables profile to default 2024-06-18T02:38:18.406 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:18 smithi163 ceph-mon[25094]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:18.406 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:18 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:18.406 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:18 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:18.406 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:18 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi186", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:38:18.406 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:18 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:18.406 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:18 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1655230867' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-06-18T02:38:18.675 INFO:tasks.cephadm:Adding mon.a on smithi163 2024-06-18T02:38:18.675 INFO:tasks.cephadm:Adding mon.c on smithi163 2024-06-18T02:38:18.675 INFO:tasks.cephadm:Adding mon.b on smithi186 2024-06-18T02:38:18.676 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch apply mon '3;smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi186:172.21.15.186=b' 2024-06-18T02:38:19.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:19 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1655230867' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-06-18T02:38:19.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:19 smithi163 ceph-mon[25094]: osdmap e4: 0 total, 0 up, 0 in 2024-06-18T02:38:20.440 INFO:teuthology.orchestra.run.smithi186.stdout:Scheduled mon update... 2024-06-18T02:38:20.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:20 smithi163 ceph-mon[25094]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:21.128 DEBUG:teuthology.orchestra.run.smithi163:mon.c> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.c.service 2024-06-18T02:38:21.130 DEBUG:teuthology.orchestra.run.smithi186:mon.b> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.b.service 2024-06-18T02:38:21.133 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-06-18T02:38:21.134 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph mon dump -f json 2024-06-18T02:38:21.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:21.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi186:172.21.15.186=b", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:38:21.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: Saving service mon spec with placement smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi186:172.21.15.186=b;count:3 2024-06-18T02:38:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T02:38:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:38:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T02:38:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:21 smithi163 ceph-mon[25094]: Deploying daemon mon.c on smithi163 2024-06-18T02:38:22.969 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-06-18T02:38:22.969 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":1,"fsid":"929affee-2d1b-11ef-bc9d-c7b262605968","modified":"2024-06-18T02:36:57.402480Z","created":"2024-06-18T02:36:57.402480Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:3300","nonce":0},{"type":"v1","addr":"172.21.15.163:6789","nonce":0}]},"addr":"172.21.15.163:6789/0","public_addr":"172.21.15.163:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-18T02:38:22.971 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 1 2024-06-18T02:38:23.765 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:23 smithi163 ceph-mon[25094]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:23.765 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:23 smithi163 ceph-mon[25094]: from='client.? 172.21.15.186:0/1657819050' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-18T02:38:24.527 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-06-18T02:38:24.527 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph mon dump -f json 2024-06-18T02:38:29.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:38:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: mon.a calling monitor election 2024-06-18T02:38:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: mon.c calling monitor election 2024-06-18T02:38:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:29.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-06-18T02:38:29.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: monmap e2: 2 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} 2024-06-18T02:38:29.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: fsmap 2024-06-18T02:38:29.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: osdmap e4: 0 total, 0 up, 0 in 2024-06-18T02:38:29.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: mgrmap e13: y(active, since 34s) 2024-06-18T02:38:29.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: overall HEALTH_OK 2024-06-18T02:38:29.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:29.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T02:38:29.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:28 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:29.343 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-06-18T02:38:29.343 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":2,"fsid":"929affee-2d1b-11ef-bc9d-c7b262605968","modified":"2024-06-18T02:38:23.932787Z","created":"2024-06-18T02:36:57.402480Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:3300","nonce":0},{"type":"v1","addr":"172.21.15.163:6789","nonce":0}]},"addr":"172.21.15.163:6789/0","public_addr":"172.21.15.163:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:3301","nonce":0},{"type":"v1","addr":"172.21.15.163:6790","nonce":0}]},"addr":"172.21.15.163:6790/0","public_addr":"172.21.15.163:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-06-18T02:38:29.345 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 2 2024-06-18T02:38:30.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:29 smithi163 ceph-mon[25094]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:30.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:29 smithi163 ceph-mon[25094]: Deploying daemon mon.b on smithi186 2024-06-18T02:38:30.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:29 smithi163 ceph-mon[25094]: from='client.? 172.21.15.186:0/3176498401' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-18T02:38:30.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:29 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:30.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:29 smithi163 ceph-mon[29791]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:30.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:29 smithi163 ceph-mon[29791]: Deploying daemon mon.b on smithi186 2024-06-18T02:38:30.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:29 smithi163 ceph-mon[29791]: from='client.? 172.21.15.186:0/3176498401' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-18T02:38:30.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:29 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:31.433 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-06-18T02:38:31.434 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph mon dump -f json 2024-06-18T02:38:32.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:32 smithi163 ceph-mon[25094]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:32.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:32 smithi163 ceph-mon[29791]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:32.727 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:32 smithi186 systemd[1]: Starting Ceph mon.b for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:38:33.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 podman[25890]: 2024-06-18 02:38:33.372531559 +0000 UTC m=+0.099355302 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-06-18T02:38:33.713 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-06-18T02:38:33.713 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":2,"fsid":"929affee-2d1b-11ef-bc9d-c7b262605968","modified":"2024-06-18T02:38:23.932787Z","created":"2024-06-18T02:36:57.402480Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:3300","nonce":0},{"type":"v1","addr":"172.21.15.163:6789","nonce":0}]},"addr":"172.21.15.163:6789/0","public_addr":"172.21.15.163:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:3301","nonce":0},{"type":"v1","addr":"172.21.15.163:6790","nonce":0}]},"addr":"172.21.15.163:6790/0","public_addr":"172.21.15.163:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-06-18T02:38:33.723 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 2 2024-06-18T02:38:33.849 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 podman[25890]: 2024-06-18 02:38:33.600079963 +0000 UTC m=+0.326903705 container create a3618f734a9fdfe387382742815495207284cce3a3e27ae27b23c8bcf40909f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, vcs-type=git, release=754, ceph=True, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, architecture=x86_64, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream) 2024-06-18T02:38:33.850 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 podman[25890]: 2024-06-18 02:38:33.826734437 +0000 UTC m=+0.553558191 container init a3618f734a9fdfe387382742815495207284cce3a3e27ae27b23c8bcf40909f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b, vcs-type=git, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, version=8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.openshift.expose-services=, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-06-18T02:38:33.850 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 podman[25890]: 2024-06-18 02:38:33.83018215 +0000 UTC m=+0.557005900 container start a3618f734a9fdfe387382742815495207284cce3a3e27ae27b23c8bcf40909f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, ceph=True, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, architecture=x86_64, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, vcs-type=git, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0) 2024-06-18T02:38:34.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: set uid:gid to 167:167 (ceph:ceph) 2024-06-18T02:38:34.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: pidfile_write: ignore empty --pid-file 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: load: jerasure load: lrc 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: RocksDB version: 6.15.5 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Compile date Apr 18 2022 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: DB SUMMARY 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: DB Session ID: P5XXXM5CER9I0YYDLLOJ 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: CURRENT file: CURRENT 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: IDENTITY file: IDENTITY 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.error_if_exists: 0 2024-06-18T02:38:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.create_if_missing: 0 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.paranoid_checks: 1 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.env: 0x564f248c6860 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.fs: Posix File System 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.info_log: 0x564f2539ddc0 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_file_opening_threads: 16 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.statistics: (nil) 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.use_fsync: 0 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_log_file_size: 0 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.keep_log_file_num: 1000 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.recycle_log_file_num: 0 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.allow_fallocate: 1 2024-06-18T02:38:34.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.allow_mmap_reads: 0 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.allow_mmap_writes: 0 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.use_direct_reads: 0 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.create_missing_column_families: 0 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.db_log_dir: 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.advise_random_on_open: 1 2024-06-18T02:38:34.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.db_write_buffer_size: 0 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.write_buffer_manager: 0x564f2548e240 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.rate_limiter: (nil) 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.wal_recovery_mode: 2 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.enable_thread_tracking: 0 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.enable_pipelined_write: 0 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.unordered_write: 0 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-18T02:38:34.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.row_cache: None 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.wal_filter: None 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.allow_ingest_behind: 0 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.preserve_deletes: 0 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.two_write_queues: 0 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.manual_wal_flush: 0 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.atomic_flush: 0 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.log_readahead_size: 0 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-18T02:38:34.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.best_efforts_recovery: 0 2024-06-18T02:38:34.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-18T02:38:34.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-18T02:38:34.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.allow_data_in_errors: 0 2024-06-18T02:38:34.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.db_host_id: __hostname__ 2024-06-18T02:38:34.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_background_jobs: 2 2024-06-18T02:38:34.325 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_background_compactions: -1 2024-06-18T02:38:34.325 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_subcompactions: 1 2024-06-18T02:38:34.325 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-18T02:38:34.325 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-18T02:38:34.325 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-18T02:38:34.325 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_total_wal_size: 0 2024-06-18T02:38:34.325 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_open_files: -1 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.bytes_per_sync: 0 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compaction_readahead_size: 0 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_background_flushes: -1 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Compression algorithms supported: 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: kZSTD supported: 0 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: kXpressCompression supported: 0 2024-06-18T02:38:34.326 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: kLZ4HCCompression supported: 1 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: kLZ4Compression supported: 1 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: kBZip2Compression supported: 0 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: kZlibCompression supported: 1 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: kSnappyCompression supported: 1 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.merge_operator: 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compaction_filter: None 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compaction_filter_factory: None 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.sst_partitioner_factory: None 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-18T02:38:34.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564f25369d00) 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: cache_index_and_filter_blocks: 1 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: pin_top_level_index_and_filter: 1 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: index_type: 0 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: data_block_index_type: 0 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: index_shortening: 1 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: hash_index_allow_collision: 1 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: checksum: 1 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: no_block_cache: 0 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_cache: 0x564f253d4170 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_cache_name: BinnedLRUCache 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_cache_options: 2024-06-18T02:38:34.328 INFO:journalctl@ceph.mon.b.smithi186.stdout: capacity : 536870912 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: num_shard_bits : 4 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: strict_capacity_limit : 0 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: high_pri_pool_ratio: 0.000 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_cache_compressed: (nil) 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: persistent_cache: (nil) 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_size: 4096 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_size_deviation: 10 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_restart_interval: 16 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: index_block_restart_interval: 1 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: metadata_block_size: 4096 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: partition_filters: 0 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: use_delta_encoding: 1 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: whole_key_filtering: 1 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: verify_compression: 0 2024-06-18T02:38:34.329 INFO:journalctl@ceph.mon.b.smithi186.stdout: read_amp_bytes_per_bit: 0 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout: format_version: 4 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout: enable_index_compression: 1 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_align: 0 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.write_buffer_size: 33554432 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_write_buffer_number: 2 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compression: NoCompression 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.bottommost_compression: Disabled 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.prefix_extractor: nullptr 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.num_levels: 7 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-18T02:38:34.330 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compression_opts.level: 32767 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compression_opts.strategy: 0 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compression_opts.enabled: false 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-18T02:38:34.331 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.target_file_size_base: 67108864 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-18T02:38:34.332 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.arena_block_size: 4194304 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.disable_auto_compactions: 0 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.table_properties_collectors: 2024-06-18T02:38:34.333 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.inplace_update_support: 0 2024-06-18T02:38:34.334 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-18T02:38:34.334 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-18T02:38:34.334 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-18T02:38:34.334 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-18T02:38:34.334 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.bloom_locality: 0 2024-06-18T02:38:34.334 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.max_successive_merges: 0 2024-06-18T02:38:34.334 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-18T02:38:34.334 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.paranoid_file_checks: 0 2024-06-18T02:38:34.334 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.force_consistency_checks: 1 2024-06-18T02:38:34.334 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.report_bg_io_stats: 0 2024-06-18T02:38:34.334 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.ttl: 2592000 2024-06-18T02:38:34.334 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-18T02:38:34.334 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.enable_blob_files: false 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.min_blob_size: 0 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.blob_file_size: 268435456 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718678313857030, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718678313857331, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1718678313, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "30036245-31ce-48f4-9d08-d025b8ce99df", "db_session_id": "P5XXXM5CER9I0YYDLLOJ"}} 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-06-18T02:38:34.335 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718678313857630, "job": 1, "event": "recovery_finished"} 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x564f253baa80 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: DB pointer 0x564f253ca000 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout: ** DB Stats ** 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout: 2024-06-18T02:38:34.336 INFO:journalctl@ceph.mon.b.smithi186.stdout: ** Compaction Stats [default] ** 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: ** Compaction Stats [default] ** 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-18T02:38:34.337 INFO:journalctl@ceph.mon.b.smithi186.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: Interval compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: ** Compaction Stats [default] ** 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-06-18T02:38:34.338 INFO:journalctl@ceph.mon.b.smithi186.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: ** Compaction Stats [default] ** 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-18T02:38:34.339 INFO:journalctl@ceph.mon.b.smithi186.stdout: 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: using public_addr v2:172.21.15.186:0/0 -> [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: starting mon.b rank -1 at public addrs [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] at bind addrs [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(???) e0 preinit fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing).mds e1 new map 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing).mds e1 print_map 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout: e1 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout: legacy client fscid: -1 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout: 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout: No filesystems configured 2024-06-18T02:38:34.340 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.a calling monitor election 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:34.341 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.c calling monitor election 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: monmap e2: 2 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: fsmap 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: osdmap e4: 0 total, 0 up, 0 in 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mgrmap e13: y(active, since 34s) 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: overall HEALTH_OK 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:34.342 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:34.343 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: Deploying daemon mon.b on smithi186 2024-06-18T02:38:34.343 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: from='client.? 172.21.15.186:0/3176498401' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-18T02:38:34.343 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:34.343 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:34.343 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-06-18T02:38:34.343 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 ceph-mon[25944]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-06-18T02:38:34.343 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 bash[25890]: a3618f734a9fdfe387382742815495207284cce3a3e27ae27b23c8bcf40909f7 2024-06-18T02:38:34.343 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:33 smithi186 systemd[1]: Started Ceph mon.b for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:38:35.519 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-06-18T02:38:35.520 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph mon dump -f json 2024-06-18T02:38:39.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: mon.a calling monitor election 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: mon.c calling monitor election 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: mon.b calling monitor election 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-18T02:38:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} 2024-06-18T02:38:39.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: fsmap 2024-06-18T02:38:39.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: osdmap e4: 0 total, 0 up, 0 in 2024-06-18T02:38:39.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: mgrmap e13: y(active, since 44s) 2024-06-18T02:38:39.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: overall HEALTH_OK 2024-06-18T02:38:39.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:39.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:39.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:39.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:39.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:38:39.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: mon.a calling monitor election 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: mon.c calling monitor election 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: mon.b calling monitor election 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: fsmap 2024-06-18T02:38:39.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: osdmap e4: 0 total, 0 up, 0 in 2024-06-18T02:38:39.296 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: mgrmap e13: y(active, since 44s) 2024-06-18T02:38:39.296 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: overall HEALTH_OK 2024-06-18T02:38:39.296 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:39.296 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:39.296 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:39.296 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:38 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: mon.a calling monitor election 2024-06-18T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: mon.c calling monitor election 2024-06-18T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: mon.b calling monitor election 2024-06-18T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:39.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-18T02:38:39.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} 2024-06-18T02:38:39.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: fsmap 2024-06-18T02:38:39.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: osdmap e4: 0 total, 0 up, 0 in 2024-06-18T02:38:39.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: mgrmap e13: y(active, since 44s) 2024-06-18T02:38:39.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: overall HEALTH_OK 2024-06-18T02:38:39.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:39.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:39.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:39.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:38 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:40.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:39 smithi163 ceph-mon[25094]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:40.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:39 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:40.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:39 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:40.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:39 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:40.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:39 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:40.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:39 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:40.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:39 smithi163 ceph-mon[29791]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:40.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:39 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:40.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:39 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:40.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:39 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:40.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:39 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:40.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:39 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:40.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:39 smithi186 ceph-mon[25944]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:40.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:39 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:40.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:39 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:40.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:39 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:40.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:39 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:38:40.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:39 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:40.339 INFO:teuthology.orchestra.run.smithi186.stdout: 2024-06-18T02:38:40.340 INFO:teuthology.orchestra.run.smithi186.stdout:{"epoch":3,"fsid":"929affee-2d1b-11ef-bc9d-c7b262605968","modified":"2024-06-18T02:38:33.877201Z","created":"2024-06-18T02:36:57.402480Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:3300","nonce":0},{"type":"v1","addr":"172.21.15.163:6789","nonce":0}]},"addr":"172.21.15.163:6789/0","public_addr":"172.21.15.163:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:3301","nonce":0},{"type":"v1","addr":"172.21.15.163:6790","nonce":0}]},"addr":"172.21.15.163:6790/0","public_addr":"172.21.15.163:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:3300","nonce":0},{"type":"v1","addr":"172.21.15.186:6789","nonce":0}]},"addr":"172.21.15.186:6789/0","public_addr":"172.21.15.186:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-06-18T02:38:40.341 INFO:teuthology.orchestra.run.smithi186.stderr:dumped monmap epoch 3 2024-06-18T02:38:40.910 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-06-18T02:38:40.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph config generate-minimal-conf 2024-06-18T02:38:41.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:41 smithi186 ceph-mon[25944]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T02:38:41.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:41 smithi186 ceph-mon[25944]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T02:38:41.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:41 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:41.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:41 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:41.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:41 smithi186 ceph-mon[25944]: Reconfiguring mon.a (unknown last config time)... 2024-06-18T02:38:41.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:41 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T02:38:41.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:41 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T02:38:41.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:41 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:41.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:41 smithi186 ceph-mon[25944]: Reconfiguring daemon mon.a on smithi163 2024-06-18T02:38:41.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:41 smithi186 ceph-mon[25944]: from='client.? 172.21.15.186:0/3842025776' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-18T02:38:41.419 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[25094]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T02:38:41.419 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[25094]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T02:38:41.419 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:41.419 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:41.419 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[25094]: Reconfiguring mon.a (unknown last config time)... 2024-06-18T02:38:41.419 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T02:38:41.420 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T02:38:41.420 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:41.420 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[25094]: Reconfiguring daemon mon.a on smithi163 2024-06-18T02:38:41.420 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[25094]: from='client.? 172.21.15.186:0/3842025776' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-18T02:38:41.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[29791]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T02:38:41.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[29791]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T02:38:41.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:41.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:41.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[29791]: Reconfiguring mon.a (unknown last config time)... 2024-06-18T02:38:41.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T02:38:41.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T02:38:41.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:41.421 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[29791]: Reconfiguring daemon mon.a on smithi163 2024-06-18T02:38:41.421 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:41 smithi163 ceph-mon[29791]: from='client.? 172.21.15.186:0/3842025776' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-18T02:38:42.440 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:42 smithi163 ceph-mon[25094]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:42.440 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:42 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:42.440 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:42 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T02:38:42.440 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:42 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T02:38:42.440 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:42 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:42.440 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:42 smithi163 ceph-mon[29791]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:42.440 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:42 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:42.440 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:42 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T02:38:42.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:42 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T02:38:42.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:42 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:42.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:42 smithi186 ceph-mon[25944]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:42.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:42 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:42.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:42 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T02:38:42.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:42 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T02:38:42.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:42 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:43.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:43 smithi163 ceph-mon[25094]: Reconfiguring mon.c (monmap changed)... 2024-06-18T02:38:43.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:43 smithi163 ceph-mon[25094]: Reconfiguring daemon mon.c on smithi163 2024-06-18T02:38:43.229 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:43 smithi163 ceph-mon[29791]: Reconfiguring mon.c (monmap changed)... 2024-06-18T02:38:43.229 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:43 smithi163 ceph-mon[29791]: Reconfiguring daemon mon.c on smithi163 2024-06-18T02:38:43.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:43 smithi186 ceph-mon[25944]: Reconfiguring mon.c (monmap changed)... 2024-06-18T02:38:43.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:43 smithi186 ceph-mon[25944]: Reconfiguring daemon mon.c on smithi163 2024-06-18T02:38:44.197 INFO:teuthology.orchestra.run.smithi163.stdout:# minimal ceph.conf for 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:38:44.197 INFO:teuthology.orchestra.run.smithi163.stdout:[global] 2024-06-18T02:38:44.197 INFO:teuthology.orchestra.run.smithi163.stdout: fsid = 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:38:44.197 INFO:teuthology.orchestra.run.smithi163.stdout: mon_host = [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] 2024-06-18T02:38:44.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:44 smithi163 ceph-mon[25094]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:44.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:44 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:44.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:44 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T02:38:44.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:44 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T02:38:44.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:44 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:44.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:44 smithi163 ceph-mon[29791]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:44.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:44 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:44.199 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:44 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T02:38:44.199 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:44 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T02:38:44.199 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:44 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:44.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:44 smithi186 ceph-mon[25944]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:44.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:44 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:44.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:44 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T02:38:44.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:44 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T02:38:44.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:44 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:44.810 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-06-18T02:38:44.811 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2024-06-18T02:38:44.811 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/ceph/ceph.conf 2024-06-18T02:38:44.839 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2024-06-18T02:38:44.839 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:38:44.905 DEBUG:teuthology.orchestra.run.smithi186:> set -ex 2024-06-18T02:38:44.905 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd of=/etc/ceph/ceph.conf 2024-06-18T02:38:44.930 DEBUG:teuthology.orchestra.run.smithi186:> set -ex 2024-06-18T02:38:44.931 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:38:44.995 INFO:tasks.cephadm:Adding mgr.y on smithi163 2024-06-18T02:38:44.995 INFO:tasks.cephadm:Adding mgr.x on smithi186 2024-06-18T02:38:44.995 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch apply mgr '2;smithi163=y;smithi186=x' 2024-06-18T02:38:45.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:45 smithi186 ceph-mon[25944]: Reconfiguring mon.b (monmap changed)... 2024-06-18T02:38:45.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:45 smithi186 ceph-mon[25944]: Reconfiguring daemon mon.b on smithi186 2024-06-18T02:38:45.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:45 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/120353550' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:45.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:45 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:45.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:45 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:45.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:45 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:45.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:45 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:45.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:45 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:45.390 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[29791]: Reconfiguring mon.b (monmap changed)... 2024-06-18T02:38:45.390 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[29791]: Reconfiguring daemon mon.b on smithi186 2024-06-18T02:38:45.390 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/120353550' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:45.391 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:45.391 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:45.391 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:45.391 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:45.391 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:45.391 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[25094]: Reconfiguring mon.b (monmap changed)... 2024-06-18T02:38:45.391 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[25094]: Reconfiguring daemon mon.b on smithi186 2024-06-18T02:38:45.391 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/120353550' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:45.391 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:45.391 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:45.391 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:45.391 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:45.392 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:45 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:46.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:46 smithi186 ceph-mon[25944]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:46.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:46 smithi163 ceph-mon[25094]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:46.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:46 smithi163 ceph-mon[29791]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:46.819 INFO:teuthology.orchestra.run.smithi186.stdout:Scheduled mgr update... 2024-06-18T02:38:47.405 DEBUG:teuthology.orchestra.run.smithi186:mgr.x> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.x.service 2024-06-18T02:38:47.407 INFO:tasks.cephadm:Deploying OSDs... 2024-06-18T02:38:47.407 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2024-06-18T02:38:47.407 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/scratch_devs of=/dev/stdout 2024-06-18T02:38:47.427 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-06-18T02:38:47.428 DEBUG:teuthology.orchestra.run.smithi163:> stat /dev/vg_nvme/lv_1 2024-06-18T02:38:47.486 INFO:teuthology.orchestra.run.smithi163.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-06-18T02:38:47.486 INFO:teuthology.orchestra.run.smithi163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-18T02:38:47.486 INFO:teuthology.orchestra.run.smithi163.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-06-18T02:38:47.487 INFO:teuthology.orchestra.run.smithi163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-18T02:38:47.487 INFO:teuthology.orchestra.run.smithi163.stdout:Context: system_u:object_r:device_t:s0 2024-06-18T02:38:47.487 INFO:teuthology.orchestra.run.smithi163.stdout:Access: 2024-06-18 02:38:23.370123892 +0000 2024-06-18T02:38:47.487 INFO:teuthology.orchestra.run.smithi163.stdout:Modify: 2024-06-18 02:35:51.356011728 +0000 2024-06-18T02:38:47.487 INFO:teuthology.orchestra.run.smithi163.stdout:Change: 2024-06-18 02:35:51.356011728 +0000 2024-06-18T02:38:47.487 INFO:teuthology.orchestra.run.smithi163.stdout: Birth: 2024-06-18 02:35:51.356011728 +0000 2024-06-18T02:38:47.487 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-06-18T02:38:47.554 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records in 2024-06-18T02:38:47.554 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records out 2024-06-18T02:38:47.554 INFO:teuthology.orchestra.run.smithi163.stderr:512 bytes copied, 0.000295227 s, 1.7 MB/s 2024-06-18T02:38:47.555 DEBUG:teuthology.orchestra.run.smithi163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-06-18T02:38:47.614 DEBUG:teuthology.orchestra.run.smithi163:> stat /dev/vg_nvme/lv_2 2024-06-18T02:38:47.671 INFO:teuthology.orchestra.run.smithi163.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-06-18T02:38:47.671 INFO:teuthology.orchestra.run.smithi163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-18T02:38:47.671 INFO:teuthology.orchestra.run.smithi163.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-06-18T02:38:47.671 INFO:teuthology.orchestra.run.smithi163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-18T02:38:47.671 INFO:teuthology.orchestra.run.smithi163.stdout:Context: system_u:object_r:device_t:s0 2024-06-18T02:38:47.671 INFO:teuthology.orchestra.run.smithi163.stdout:Access: 2024-06-18 02:38:23.370123892 +0000 2024-06-18T02:38:47.671 INFO:teuthology.orchestra.run.smithi163.stdout:Modify: 2024-06-18 02:35:51.793001912 +0000 2024-06-18T02:38:47.671 INFO:teuthology.orchestra.run.smithi163.stdout:Change: 2024-06-18 02:35:51.793001912 +0000 2024-06-18T02:38:47.671 INFO:teuthology.orchestra.run.smithi163.stdout: Birth: 2024-06-18 02:35:51.793001912 +0000 2024-06-18T02:38:47.672 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-06-18T02:38:47.736 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records in 2024-06-18T02:38:47.736 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records out 2024-06-18T02:38:47.736 INFO:teuthology.orchestra.run.smithi163.stderr:512 bytes copied, 0.000218875 s, 2.3 MB/s 2024-06-18T02:38:47.737 DEBUG:teuthology.orchestra.run.smithi163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-06-18T02:38:47.794 DEBUG:teuthology.orchestra.run.smithi163:> stat /dev/vg_nvme/lv_3 2024-06-18T02:38:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-06-18T02:38:47.851 INFO:teuthology.orchestra.run.smithi163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-18T02:38:47.851 INFO:teuthology.orchestra.run.smithi163.stdout:Device: 5h/5d Inode: 708 Links: 1 2024-06-18T02:38:47.851 INFO:teuthology.orchestra.run.smithi163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-18T02:38:47.851 INFO:teuthology.orchestra.run.smithi163.stdout:Context: system_u:object_r:device_t:s0 2024-06-18T02:38:47.851 INFO:teuthology.orchestra.run.smithi163.stdout:Access: 2024-06-18 02:38:23.371123869 +0000 2024-06-18T02:38:47.851 INFO:teuthology.orchestra.run.smithi163.stdout:Modify: 2024-06-18 02:35:52.121994515 +0000 2024-06-18T02:38:47.851 INFO:teuthology.orchestra.run.smithi163.stdout:Change: 2024-06-18 02:35:52.121994515 +0000 2024-06-18T02:38:47.851 INFO:teuthology.orchestra.run.smithi163.stdout: Birth: 2024-06-18 02:35:52.121994515 +0000 2024-06-18T02:38:47.851 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-06-18T02:38:47.902 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:47 smithi186 ceph-mon[25944]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:47.902 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:47 smithi186 ceph-mon[25944]: from='client.24107 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi163=y;smithi186=x", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:38:47.902 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:47 smithi186 ceph-mon[25944]: Saving service mgr spec with placement smithi163=y;smithi186=x;count:2 2024-06-18T02:38:47.902 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:47 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:47.902 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:47 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:47.902 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:47 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:47.902 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:47 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:47.902 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:47 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:47.903 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:47 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:38:47.903 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:47 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-06-18T02:38:47.903 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:47 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T02:38:47.903 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:47 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:47.903 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:47 smithi186 ceph-mon[25944]: Deploying daemon mgr.x on smithi186 2024-06-18T02:38:47.914 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records in 2024-06-18T02:38:47.914 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records out 2024-06-18T02:38:47.915 INFO:teuthology.orchestra.run.smithi163.stderr:512 bytes copied, 0.000238244 s, 2.1 MB/s 2024-06-18T02:38:47.916 DEBUG:teuthology.orchestra.run.smithi163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-06-18T02:38:47.973 DEBUG:teuthology.orchestra.run.smithi163:> stat /dev/vg_nvme/lv_4 2024-06-18T02:38:48.029 INFO:teuthology.orchestra.run.smithi163.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-06-18T02:38:48.029 INFO:teuthology.orchestra.run.smithi163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-18T02:38:48.029 INFO:teuthology.orchestra.run.smithi163.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-06-18T02:38:48.029 INFO:teuthology.orchestra.run.smithi163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-18T02:38:48.029 INFO:teuthology.orchestra.run.smithi163.stdout:Context: system_u:object_r:device_t:s0 2024-06-18T02:38:48.029 INFO:teuthology.orchestra.run.smithi163.stdout:Access: 2024-06-18 02:38:23.371123869 +0000 2024-06-18T02:38:48.029 INFO:teuthology.orchestra.run.smithi163.stdout:Modify: 2024-06-18 02:35:52.465986785 +0000 2024-06-18T02:38:48.029 INFO:teuthology.orchestra.run.smithi163.stdout:Change: 2024-06-18 02:35:52.465986785 +0000 2024-06-18T02:38:48.029 INFO:teuthology.orchestra.run.smithi163.stdout: Birth: 2024-06-18 02:35:52.465986785 +0000 2024-06-18T02:38:48.030 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-06-18T02:38:48.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[29791]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:48.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[29791]: from='client.24107 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi163=y;smithi186=x", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:38:48.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[29791]: Saving service mgr spec with placement smithi163=y;smithi186=x;count:2 2024-06-18T02:38:48.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:48.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:48.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:48.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:48.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:48.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:38:48.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-06-18T02:38:48.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[29791]: Deploying daemon mgr.x on smithi186 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[25094]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[25094]: from='client.24107 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi163=y;smithi186=x", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[25094]: Saving service mgr spec with placement smithi163=y;smithi186=x;count:2 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T02:38:48.092 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:48.093 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:47 smithi163 ceph-mon[25094]: Deploying daemon mgr.x on smithi186 2024-06-18T02:38:48.093 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records in 2024-06-18T02:38:48.093 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records out 2024-06-18T02:38:48.093 INFO:teuthology.orchestra.run.smithi163.stderr:512 bytes copied, 0.000189841 s, 2.7 MB/s 2024-06-18T02:38:48.094 DEBUG:teuthology.orchestra.run.smithi163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-06-18T02:38:48.151 DEBUG:teuthology.orchestra.run.smithi186:> set -ex 2024-06-18T02:38:48.151 DEBUG:teuthology.orchestra.run.smithi186:> dd if=/scratch_devs of=/dev/stdout 2024-06-18T02:38:48.167 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-06-18T02:38:48.167 DEBUG:teuthology.orchestra.run.smithi186:> stat /dev/vg_nvme/lv_1 2024-06-18T02:38:48.223 INFO:teuthology.orchestra.run.smithi186.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-06-18T02:38:48.223 INFO:teuthology.orchestra.run.smithi186.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-18T02:38:48.223 INFO:teuthology.orchestra.run.smithi186.stdout:Device: 5h/5d Inode: 675 Links: 1 2024-06-18T02:38:48.223 INFO:teuthology.orchestra.run.smithi186.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-18T02:38:48.223 INFO:teuthology.orchestra.run.smithi186.stdout:Context: system_u:object_r:device_t:s0 2024-06-18T02:38:48.223 INFO:teuthology.orchestra.run.smithi186.stdout:Access: 2024-06-18 02:38:32.576496394 +0000 2024-06-18T02:38:48.223 INFO:teuthology.orchestra.run.smithi186.stdout:Modify: 2024-06-18 02:35:51.364384580 +0000 2024-06-18T02:38:48.223 INFO:teuthology.orchestra.run.smithi186.stdout:Change: 2024-06-18 02:35:51.364384580 +0000 2024-06-18T02:38:48.223 INFO:teuthology.orchestra.run.smithi186.stdout: Birth: 2024-06-18 02:35:51.364384580 +0000 2024-06-18T02:38:48.224 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-06-18T02:38:48.287 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records in 2024-06-18T02:38:48.287 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records out 2024-06-18T02:38:48.287 INFO:teuthology.orchestra.run.smithi186.stderr:512 bytes copied, 0.00018672 s, 2.7 MB/s 2024-06-18T02:38:48.288 DEBUG:teuthology.orchestra.run.smithi186:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-06-18T02:38:48.344 DEBUG:teuthology.orchestra.run.smithi186:> stat /dev/vg_nvme/lv_2 2024-06-18T02:38:48.400 INFO:teuthology.orchestra.run.smithi186.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-06-18T02:38:48.400 INFO:teuthology.orchestra.run.smithi186.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-18T02:38:48.400 INFO:teuthology.orchestra.run.smithi186.stdout:Device: 5h/5d Inode: 693 Links: 1 2024-06-18T02:38:48.400 INFO:teuthology.orchestra.run.smithi186.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-18T02:38:48.400 INFO:teuthology.orchestra.run.smithi186.stdout:Context: system_u:object_r:device_t:s0 2024-06-18T02:38:48.400 INFO:teuthology.orchestra.run.smithi186.stdout:Access: 2024-06-18 02:38:32.576496394 +0000 2024-06-18T02:38:48.400 INFO:teuthology.orchestra.run.smithi186.stdout:Modify: 2024-06-18 02:35:51.693379023 +0000 2024-06-18T02:38:48.400 INFO:teuthology.orchestra.run.smithi186.stdout:Change: 2024-06-18 02:35:51.693379023 +0000 2024-06-18T02:38:48.400 INFO:teuthology.orchestra.run.smithi186.stdout: Birth: 2024-06-18 02:35:51.693379023 +0000 2024-06-18T02:38:48.401 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-06-18T02:38:48.593 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records in 2024-06-18T02:38:48.593 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records out 2024-06-18T02:38:48.593 INFO:teuthology.orchestra.run.smithi186.stderr:512 bytes copied, 0.000183781 s, 2.8 MB/s 2024-06-18T02:38:48.595 DEBUG:teuthology.orchestra.run.smithi186:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-06-18T02:38:48.611 DEBUG:teuthology.orchestra.run.smithi186:> stat /dev/vg_nvme/lv_3 2024-06-18T02:38:48.670 INFO:teuthology.orchestra.run.smithi186.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-06-18T02:38:48.670 INFO:teuthology.orchestra.run.smithi186.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-18T02:38:48.670 INFO:teuthology.orchestra.run.smithi186.stdout:Device: 5h/5d Inode: 711 Links: 1 2024-06-18T02:38:48.670 INFO:teuthology.orchestra.run.smithi186.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-18T02:38:48.670 INFO:teuthology.orchestra.run.smithi186.stdout:Context: system_u:object_r:device_t:s0 2024-06-18T02:38:48.671 INFO:teuthology.orchestra.run.smithi186.stdout:Access: 2024-06-18 02:38:48.553229554 +0000 2024-06-18T02:38:48.671 INFO:teuthology.orchestra.run.smithi186.stdout:Modify: 2024-06-18 02:35:52.024373432 +0000 2024-06-18T02:38:48.671 INFO:teuthology.orchestra.run.smithi186.stdout:Change: 2024-06-18 02:35:52.024373432 +0000 2024-06-18T02:38:48.671 INFO:teuthology.orchestra.run.smithi186.stdout: Birth: 2024-06-18 02:35:52.024373432 +0000 2024-06-18T02:38:48.671 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-06-18T02:38:48.768 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records in 2024-06-18T02:38:48.769 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records out 2024-06-18T02:38:48.769 INFO:teuthology.orchestra.run.smithi186.stderr:512 bytes copied, 0.000209024 s, 2.4 MB/s 2024-06-18T02:38:48.770 DEBUG:teuthology.orchestra.run.smithi186:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-06-18T02:38:48.786 DEBUG:teuthology.orchestra.run.smithi186:> stat /dev/vg_nvme/lv_4 2024-06-18T02:38:48.842 INFO:teuthology.orchestra.run.smithi186.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-06-18T02:38:48.842 INFO:teuthology.orchestra.run.smithi186.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-18T02:38:48.842 INFO:teuthology.orchestra.run.smithi186.stdout:Device: 5h/5d Inode: 732 Links: 1 2024-06-18T02:38:48.842 INFO:teuthology.orchestra.run.smithi186.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-18T02:38:48.842 INFO:teuthology.orchestra.run.smithi186.stdout:Context: system_u:object_r:device_t:s0 2024-06-18T02:38:48.842 INFO:teuthology.orchestra.run.smithi186.stdout:Access: 2024-06-18 02:38:48.729226549 +0000 2024-06-18T02:38:48.842 INFO:teuthology.orchestra.run.smithi186.stdout:Modify: 2024-06-18 02:35:52.366367656 +0000 2024-06-18T02:38:48.842 INFO:teuthology.orchestra.run.smithi186.stdout:Change: 2024-06-18 02:35:52.366367656 +0000 2024-06-18T02:38:48.842 INFO:teuthology.orchestra.run.smithi186.stdout: Birth: 2024-06-18 02:35:52.366367656 +0000 2024-06-18T02:38:48.842 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-06-18T02:38:48.903 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:48 smithi186 systemd[1]: Starting Ceph mgr.x for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:38:48.907 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records in 2024-06-18T02:38:48.907 INFO:teuthology.orchestra.run.smithi186.stderr:1+0 records out 2024-06-18T02:38:48.907 INFO:teuthology.orchestra.run.smithi186.stderr:512 bytes copied, 0.000284586 s, 1.8 MB/s 2024-06-18T02:38:48.908 DEBUG:teuthology.orchestra.run.smithi186:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-06-18T02:38:48.965 INFO:tasks.cephadm:Deploying osd.0 on smithi163 with /dev/vg_nvme/lv_4... 2024-06-18T02:38:48.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-06-18T02:38:49.170 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:49 smithi186 podman[27064]: 2024-06-18 02:38:48.912686212 +0000 UTC m=+0.018782496 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-06-18T02:38:49.170 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:49 smithi186 podman[27064]: 2024-06-18 02:38:49.038803283 +0000 UTC m=+0.144899566 container create d7519f47d8b0573212e8da1d90f13e4d64c2ee1d5c50664d066b473ad56cb3ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, release=754, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vcs-type=git, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-06-18T02:38:49.539 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:49 smithi186 podman[27064]: 2024-06-18 02:38:49.258519425 +0000 UTC m=+0.364615709 container init d7519f47d8b0573212e8da1d90f13e4d64c2ee1d5c50664d066b473ad56cb3ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, ceph=True, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, vendor=Red Hat, Inc., release=754, build-date=2022-05-03T08:36:31.336870, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , version=8, GIT_BRANCH=HEAD) 2024-06-18T02:38:49.540 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:49 smithi186 podman[27064]: 2024-06-18 02:38:49.261420924 +0000 UTC m=+0.367517211 container start d7519f47d8b0573212e8da1d90f13e4d64c2ee1d5c50664d066b473ad56cb3ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.openshift.expose-services=, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vendor=Red Hat, Inc., release=754, GIT_CLEAN=True) 2024-06-18T02:38:49.540 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:49 smithi186 ceph-mgr[27115]: -- 172.21.15.186:0/2644103619 <== mon.2 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55cb37512340 con 0x55cb374eac00 2024-06-18T02:38:49.540 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:49 smithi186 bash[27064]: d7519f47d8b0573212e8da1d90f13e4d64c2ee1d5c50664d066b473ad56cb3ab 2024-06-18T02:38:49.540 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:49 smithi186 systemd[1]: Started Ceph mgr.x for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:38:49.540 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:49.405+0000 7fab65a26000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T02:38:49.540 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:49.539+0000 7fab65a26000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:38:49.817 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:49.646+0000 7fab65a26000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T02:38:50.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:49 smithi163 ceph-mon[25094]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:50.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:49 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:50.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:49 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:50.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:49 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:50.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:49 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:50.194 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:49 smithi163 ceph-mon[29791]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:50.194 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:49 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:50.194 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:49 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:50.194 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:49 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:50.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:49 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:50.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:49 smithi186 ceph-mon[25944]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:50.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:49 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:50.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:49 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:50.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:49 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:50.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:49 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:50.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:49.974+0000 7fab65a26000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T02:38:50.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:50 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:50.059+0000 7fab65a26000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T02:38:51.044 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-06-18T02:38:51.044 INFO:teuthology.orchestra.run.smithi163.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-06-18T02:38:51.044 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10+0 records in 2024-06-18T02:38:51.045 INFO:teuthology.orchestra.run.smithi163.stderr:10+0 records out 2024-06-18T02:38:51.045 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.013098 s, 801 MB/s 2024-06-18T02:38:51.045 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping successful for: 2024-06-18T02:38:51.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:51.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[29791]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:51.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:51.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[29791]: Reconfiguring mgr.y (unknown last config time)... 2024-06-18T02:38:51.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:38:51.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T02:38:51.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:51.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[29791]: Reconfiguring daemon mgr.y on smithi163 2024-06-18T02:38:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[25094]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[25094]: Reconfiguring mgr.y (unknown last config time)... 2024-06-18T02:38:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:38:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T02:38:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:51 smithi163 ceph-mon[25094]: Reconfiguring daemon mgr.y on smithi163 2024-06-18T02:38:51.567 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:51.201+0000 7fab65a26000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T02:38:51.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:51.362+0000 7fab65a26000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T02:38:51.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:51.443+0000 7fab65a26000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T02:38:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-mon[25944]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-mon[25944]: Reconfiguring mgr.y (unknown last config time)... 2024-06-18T02:38:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:38:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T02:38:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-mon[25944]: Reconfiguring daemon mgr.y on smithi163 2024-06-18T02:38:52.067 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:51.756+0000 7fab65a26000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T02:38:52.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:51.846+0000 7fab65a26000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T02:38:52.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:51.937+0000 7fab65a26000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T02:38:52.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi163:vg_nvme/lv_4 2024-06-18T02:38:52.567 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:52.176+0000 7fab65a26000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:38:53.067 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:52.609+0000 7fab65a26000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T02:38:53.473 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:53 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:53.473 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:53 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:53.473 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:53 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:53.473 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:53 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:53.473 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:53 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:53.473 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:53.091+0000 7fab65a26000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T02:38:53.473 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:53.182+0000 7fab65a26000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T02:38:53.473 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:53.473+0000 7fab65a26000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T02:38:53.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:53 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:53.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:53 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:53.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:53 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:53.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:53 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:53.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:53 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:53.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:53 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:53.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:53 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:38:53.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:53 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:53.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:53 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:38:53.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:53 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:53.817 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:53.560+0000 7fab65a26000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T02:38:54.309 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:54 smithi186 ceph-mon[25944]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:54.309 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:53.976+0000 7fab65a26000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T02:38:54.309 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:54 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:54.105+0000 7fab65a26000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T02:38:54.309 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:54 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:54.197+0000 7fab65a26000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T02:38:54.309 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:54 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:54.309+0000 7fab65a26000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T02:38:54.416 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:54 smithi163 ceph-mon[25094]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:54.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:54 smithi163 ceph-mon[29791]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:54.567 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:54 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:54.397+0000 7fab65a26000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T02:38:55.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:54 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:54.938+0000 7fab65a26000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T02:38:55.298 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:55 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:38:55.298 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:55 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:38:55.298 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:55 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:55.298 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:55 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:38:55.299 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:55 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:38:55.299 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:55 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:55.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:55 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:38:55.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:55 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:38:55.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:55 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:55.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:55 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:38:55.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:55 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:38:55.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:55 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:55.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:38:55.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:38:55.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:38:55.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:38:55.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:38:55.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:38:55.567 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:38:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:38:55.298+0000 7fab65a26000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T02:38:56.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[29791]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:56.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[29791]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:38:56.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[29791]: Standby manager daemon x started 2024-06-18T02:38:56.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/897468741' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T02:38:56.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/897468741' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:38:56.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/897468741' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T02:38:56.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/897468741' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:38:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[25094]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[25094]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:38:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[25094]: Standby manager daemon x started 2024-06-18T02:38:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/897468741' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T02:38:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/897468741' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:38:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/897468741' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T02:38:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:56 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/897468741' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:38:56.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:56 smithi186 ceph-mon[25944]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:56 smithi186 ceph-mon[25944]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:38:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:56 smithi186 ceph-mon[25944]: Standby manager daemon x started 2024-06-18T02:38:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:56 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/897468741' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T02:38:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:56 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/897468741' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:38:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:56 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/897468741' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T02:38:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:56 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/897468741' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:38:57.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:57 smithi163 ceph-mon[25094]: mgrmap e14: y(active, since 61s), standbys: x 2024-06-18T02:38:57.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T02:38:57.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:57 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1035268515' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "142df380-e27f-4351-b05c-367d07947ced"}]: dispatch 2024-06-18T02:38:57.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:57 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1035268515' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "142df380-e27f-4351-b05c-367d07947ced"}]': finished 2024-06-18T02:38:57.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:57 smithi163 ceph-mon[25094]: osdmap e5: 1 total, 0 up, 1 in 2024-06-18T02:38:57.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:38:57.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:57 smithi163 ceph-mon[29791]: mgrmap e14: y(active, since 61s), standbys: x 2024-06-18T02:38:57.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:57 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T02:38:57.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:57 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1035268515' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "142df380-e27f-4351-b05c-367d07947ced"}]: dispatch 2024-06-18T02:38:57.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:57 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1035268515' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "142df380-e27f-4351-b05c-367d07947ced"}]': finished 2024-06-18T02:38:57.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:57 smithi163 ceph-mon[29791]: osdmap e5: 1 total, 0 up, 1 in 2024-06-18T02:38:57.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:57 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:38:57.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:57 smithi186 ceph-mon[25944]: mgrmap e14: y(active, since 61s), standbys: x 2024-06-18T02:38:57.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:57 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T02:38:57.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:57 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1035268515' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "142df380-e27f-4351-b05c-367d07947ced"}]: dispatch 2024-06-18T02:38:57.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:57 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1035268515' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "142df380-e27f-4351-b05c-367d07947ced"}]': finished 2024-06-18T02:38:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:57 smithi186 ceph-mon[25944]: osdmap e5: 1 total, 0 up, 1 in 2024-06-18T02:38:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:57 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:38:58.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:58 smithi163 ceph-mon[25094]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:58.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:38:58 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/4087460589' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:38:58.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:58 smithi163 ceph-mon[29791]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:58.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:38:58 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/4087460589' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:38:58.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:58 smithi186 ceph-mon[25944]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:38:58.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:38:58 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/4087460589' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:39:00.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:00 smithi163 ceph-mon[25094]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:00.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:00 smithi163 ceph-mon[29791]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:00.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:00 smithi186 ceph-mon[25944]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:02.224 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:02 smithi163 ceph-mon[29791]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:02.224 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:02 smithi163 ceph-mon[25094]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:02.517 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:02 smithi186 ceph-mon[25944]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:03.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:03 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-18T02:39:03.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:03 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:03.228 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:03 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-18T02:39:03.228 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:03 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:03.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:03 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-18T02:39:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:03 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:04.472 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:04 smithi163 ceph-mon[29791]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:04.473 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:04 smithi163 ceph-mon[29791]: Deploying daemon osd.0 on smithi163 2024-06-18T02:39:04.473 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:04 smithi163 ceph-mon[25094]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:04.473 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:04 smithi163 ceph-mon[25094]: Deploying daemon osd.0 on smithi163 2024-06-18T02:39:04.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:04 smithi186 ceph-mon[25944]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:04.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:04 smithi186 ceph-mon[25944]: Deploying daemon osd.0 on smithi163 2024-06-18T02:39:06.535 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:06 smithi163 ceph-mon[25094]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:06.535 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:06 smithi163 ceph-mon[29791]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:06.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:06 smithi186 ceph-mon[25944]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:08.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:08 smithi163 ceph-mon[25094]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:08.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:08 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:08.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:08 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:08.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:08 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:08.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:08 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:08.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:08 smithi163 ceph-mon[29791]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:08.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:08 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:08.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:08 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:08.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:08 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:08.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:08 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:08.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:08 smithi186 ceph-mon[25944]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:08.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:08 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:08.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:08 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:08.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:08 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:08.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:08 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:10.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:10 smithi163 ceph-mon[25094]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:10.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:10 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:10.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:10 smithi163 ceph-mon[29791]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:10.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:10 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:10.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:10 smithi186 ceph-mon[25944]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:10.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:10 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:10.993 INFO:teuthology.orchestra.run.smithi163.stdout:Created osd(s) 0 on host 'smithi163' 2024-06-18T02:39:11.580 DEBUG:teuthology.orchestra.run.smithi163:osd.0> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.0.service 2024-06-18T02:39:11.582 INFO:tasks.cephadm:Deploying osd.1 on smithi163 with /dev/vg_nvme/lv_3... 2024-06-18T02:39:11.582 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-06-18T02:39:11.644 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:11.644 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[29791]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:11.644 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[29791]: from='osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-18T02:39:11.644 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:11.645 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:11.645 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:11.645 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:11.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:11.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[25094]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:11.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[25094]: from='osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-18T02:39:11.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:11.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:11.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:11.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:11 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:11.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:11 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:11.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:11 smithi186 ceph-mon[25944]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:11.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:11 smithi186 ceph-mon[25944]: from='osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-18T02:39:11.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:11 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:11.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:11 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:11.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:11 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:11.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:11 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:12.614 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:12 smithi163 ceph-mon[25094]: from='osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-18T02:39:12.614 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:12 smithi163 ceph-mon[25094]: osdmap e6: 1 total, 0 up, 1 in 2024-06-18T02:39:12.614 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:12 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:39:12.614 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:12 smithi163 ceph-mon[25094]: from='osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T02:39:12.615 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:12 smithi163 ceph-mon[29791]: from='osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-18T02:39:12.615 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:12 smithi163 ceph-mon[29791]: osdmap e6: 1 total, 0 up, 1 in 2024-06-18T02:39:12.615 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:12 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:39:12.615 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:12 smithi163 ceph-mon[29791]: from='osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T02:39:12.615 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 02:39:12 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0[32629]: 2024-06-18T02:39:12.381+0000 7f6650b93700 -1 osd.0 0 waiting for initial osdmap 2024-06-18T02:39:12.615 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 02:39:12 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0[32629]: 2024-06-18T02:39:12.384+0000 7f664bd2b700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T02:39:12.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:12 smithi186 ceph-mon[25944]: from='osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-18T02:39:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:12 smithi186 ceph-mon[25944]: osdmap e6: 1 total, 0 up, 1 in 2024-06-18T02:39:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:12 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:39:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:12 smithi186 ceph-mon[25944]: from='osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T02:39:13.683 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:13 smithi163 ceph-mon[29791]: from='osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2024-06-18T02:39:13.683 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:13 smithi163 ceph-mon[29791]: osdmap e7: 1 total, 0 up, 1 in 2024-06-18T02:39:13.683 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:13 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:39:13.683 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:13 smithi163 ceph-mon[29791]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:13.683 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:13 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:39:13.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:13 smithi163 ceph-mon[25094]: from='osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2024-06-18T02:39:13.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:13 smithi163 ceph-mon[25094]: osdmap e7: 1 total, 0 up, 1 in 2024-06-18T02:39:13.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:13 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:39:13.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:13 smithi163 ceph-mon[25094]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:13.684 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:13 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:39:13.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:13 smithi186 ceph-mon[25944]: from='osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2024-06-18T02:39:13.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:13 smithi186 ceph-mon[25944]: osdmap e7: 1 total, 0 up, 1 in 2024-06-18T02:39:13.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:13 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:39:13.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:13 smithi186 ceph-mon[25944]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:13.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:13 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:39:14.410 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-06-18T02:39:14.410 INFO:teuthology.orchestra.run.smithi163.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-06-18T02:39:14.411 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10+0 records in 2024-06-18T02:39:14.411 INFO:teuthology.orchestra.run.smithi163.stderr:10+0 records out 2024-06-18T02:39:14.411 INFO:teuthology.orchestra.run.smithi163.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0178713 s, 587 MB/s 2024-06-18T02:39:14.411 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping successful for: 2024-06-18T02:39:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:14 smithi163 ceph-mon[29791]: purged_snaps scrub starts 2024-06-18T02:39:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:14 smithi163 ceph-mon[29791]: purged_snaps scrub ok 2024-06-18T02:39:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:14 smithi163 ceph-mon[29791]: osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344] boot 2024-06-18T02:39:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:14 smithi163 ceph-mon[29791]: osdmap e8: 1 total, 1 up, 1 in 2024-06-18T02:39:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:14 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:39:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:14 smithi163 ceph-mon[25094]: purged_snaps scrub starts 2024-06-18T02:39:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:14 smithi163 ceph-mon[25094]: purged_snaps scrub ok 2024-06-18T02:39:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:14 smithi163 ceph-mon[25094]: osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344] boot 2024-06-18T02:39:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:14 smithi163 ceph-mon[25094]: osdmap e8: 1 total, 1 up, 1 in 2024-06-18T02:39:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:14 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:39:14.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:14 smithi186 ceph-mon[25944]: purged_snaps scrub starts 2024-06-18T02:39:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:14 smithi186 ceph-mon[25944]: purged_snaps scrub ok 2024-06-18T02:39:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:14 smithi186 ceph-mon[25944]: osd.0 [v2:172.21.15.163:6802/4251995344,v1:172.21.15.163:6803/4251995344] boot 2024-06-18T02:39:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:14 smithi186 ceph-mon[25944]: osdmap e8: 1 total, 1 up, 1 in 2024-06-18T02:39:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:14 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:39:15.066 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi163:vg_nvme/lv_3 2024-06-18T02:39:15.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:15 smithi163 ceph-mon[25094]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:15.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:15 smithi163 ceph-mon[25094]: osdmap e9: 1 total, 1 up, 1 in 2024-06-18T02:39:15.683 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:15 smithi163 ceph-mon[29791]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:15.683 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:15 smithi163 ceph-mon[29791]: osdmap e9: 1 total, 1 up, 1 in 2024-06-18T02:39:15.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:15 smithi186 ceph-mon[25944]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-18T02:39:15.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:15 smithi186 ceph-mon[25944]: osdmap e9: 1 total, 1 up, 1 in 2024-06-18T02:39:16.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:16 smithi163 ceph-mon[29791]: Detected new or changed devices on smithi163 2024-06-18T02:39:16.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:16 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:16.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:16 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:39:16.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:16 smithi163 ceph-mon[29791]: Adjusting osd_memory_target on smithi163 to 16064M 2024-06-18T02:39:16.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:16 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:16.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:16 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:16.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:16 smithi163 ceph-mon[25094]: Detected new or changed devices on smithi163 2024-06-18T02:39:16.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:16 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:16.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:16 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:39:16.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:16 smithi163 ceph-mon[25094]: Adjusting osd_memory_target on smithi163 to 16064M 2024-06-18T02:39:16.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:16 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:16.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:16 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:17.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:16 smithi186 ceph-mon[25944]: Detected new or changed devices on smithi163 2024-06-18T02:39:17.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:16 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:17.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:16 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:39:17.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:16 smithi186 ceph-mon[25944]: Adjusting osd_memory_target on smithi163 to 16064M 2024-06-18T02:39:17.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:16 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:17.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:16 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:18.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:17 smithi163 ceph-mon[29791]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:18.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:17 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:39:18.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:17 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:39:18.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:17 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:18.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:17 smithi163 ceph-mon[25094]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:18.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:17 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:39:18.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:17 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:39:18.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:17 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:18.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:17 smithi186 ceph-mon[25944]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:18.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:17 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:39:18.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:17 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:39:18.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:17 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:19.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:18 smithi163 ceph-mon[25094]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:39:19.017 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:18 smithi163 ceph-mon[29791]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:39:19.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:18 smithi186 ceph-mon[25944]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:39:20.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:19 smithi163 ceph-mon[25094]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:20.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:19 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2028058096' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "696b47f4-f00e-4c37-80ec-6df94769a34f"}]: dispatch 2024-06-18T02:39:20.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:19 smithi163 ceph-mon[25094]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "696b47f4-f00e-4c37-80ec-6df94769a34f"}]: dispatch 2024-06-18T02:39:20.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:19 smithi163 ceph-mon[25094]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "696b47f4-f00e-4c37-80ec-6df94769a34f"}]': finished 2024-06-18T02:39:20.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:19 smithi163 ceph-mon[25094]: osdmap e10: 2 total, 1 up, 2 in 2024-06-18T02:39:20.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:19 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:20.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:19 smithi163 ceph-mon[29791]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:20.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:19 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2028058096' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "696b47f4-f00e-4c37-80ec-6df94769a34f"}]: dispatch 2024-06-18T02:39:20.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:19 smithi163 ceph-mon[29791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "696b47f4-f00e-4c37-80ec-6df94769a34f"}]: dispatch 2024-06-18T02:39:20.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:19 smithi163 ceph-mon[29791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "696b47f4-f00e-4c37-80ec-6df94769a34f"}]': finished 2024-06-18T02:39:20.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:19 smithi163 ceph-mon[29791]: osdmap e10: 2 total, 1 up, 2 in 2024-06-18T02:39:20.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:19 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:20.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:19 smithi186 ceph-mon[25944]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:20.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:19 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2028058096' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "696b47f4-f00e-4c37-80ec-6df94769a34f"}]: dispatch 2024-06-18T02:39:20.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:19 smithi186 ceph-mon[25944]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "696b47f4-f00e-4c37-80ec-6df94769a34f"}]: dispatch 2024-06-18T02:39:20.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:19 smithi186 ceph-mon[25944]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "696b47f4-f00e-4c37-80ec-6df94769a34f"}]': finished 2024-06-18T02:39:20.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:19 smithi186 ceph-mon[25944]: osdmap e10: 2 total, 1 up, 2 in 2024-06-18T02:39:20.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:19 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:21.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:21 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/17399274' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:39:21.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:21 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/17399274' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:39:21.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:21 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/17399274' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:39:22.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:22 smithi163 ceph-mon[25094]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:22.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:22 smithi163 ceph-mon[29791]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:22.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:22 smithi186 ceph-mon[25944]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:24.236 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:24 smithi163 ceph-mon[29791]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:24.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:24 smithi163 ceph-mon[25094]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:24.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:24 smithi186 ceph-mon[25944]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:26.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:26 smithi163 ceph-mon[25094]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:26.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:26 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-18T02:39:26.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:26 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:26.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:26 smithi163 ceph-mon[29791]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:26.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:26 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-18T02:39:26.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:26 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:26.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:26 smithi186 ceph-mon[25944]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:26.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:26 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-18T02:39:26.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:26 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:27.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:27 smithi163 ceph-mon[25094]: Deploying daemon osd.1 on smithi163 2024-06-18T02:39:27.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:27 smithi163 ceph-mon[29791]: Deploying daemon osd.1 on smithi163 2024-06-18T02:39:27.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:27 smithi186 ceph-mon[25944]: Deploying daemon osd.1 on smithi163 2024-06-18T02:39:28.202 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:28 smithi163 ceph-mon[25094]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:28.202 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:28 smithi163 ceph-mon[29791]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:28.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:28 smithi186 ceph-mon[25944]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:30.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:30 smithi186 ceph-mon[25944]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:30.373 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:30 smithi163 ceph-mon[25094]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:30.373 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:30 smithi163 ceph-mon[29791]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:31.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:31 smithi186 ceph-mon[25944]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:31.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:31 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:31.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:31 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:31.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:31 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:31.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:31 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:31.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:31 smithi163 ceph-mon[25094]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:31.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:31 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:31.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:31 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:31.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:31 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:31.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:31 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:31.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:31 smithi163 ceph-mon[29791]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:31.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:31 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:31.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:31 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:31.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:31 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:31.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:31 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:33.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:33 smithi163 ceph-mon[29791]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:33.498 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:33 smithi163 ceph-mon[25094]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:33.529 INFO:teuthology.orchestra.run.smithi163.stdout:Created osd(s) 1 on host 'smithi163' 2024-06-18T02:39:33.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:33 smithi186 ceph-mon[25944]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:34.091 DEBUG:teuthology.orchestra.run.smithi163:osd.1> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.1.service 2024-06-18T02:39:34.093 INFO:tasks.cephadm:Deploying osd.2 on smithi163 with /dev/vg_nvme/lv_2... 2024-06-18T02:39:34.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-06-18T02:39:34.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:34 smithi163 ceph-mon[29791]: from='osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-18T02:39:34.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:34 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:34.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:34 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:34.676 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:34 smithi163 ceph-mon[25094]: from='osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-18T02:39:34.676 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:34 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:34.676 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:34 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:34 smithi186 ceph-mon[25944]: from='osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-18T02:39:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:34 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:34 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:35.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:35 smithi163 ceph-mon[29791]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:35.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:35 smithi163 ceph-mon[29791]: from='osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-18T02:39:35.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:35 smithi163 ceph-mon[29791]: osdmap e11: 2 total, 1 up, 2 in 2024-06-18T02:39:35.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:35 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:35.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:35 smithi163 ceph-mon[29791]: from='osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T02:39:35.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:35 smithi163 ceph-mon[25094]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:35.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:35 smithi163 ceph-mon[25094]: from='osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-18T02:39:35.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:35 smithi163 ceph-mon[25094]: osdmap e11: 2 total, 1 up, 2 in 2024-06-18T02:39:35.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:35 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:35.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:35 smithi163 ceph-mon[25094]: from='osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T02:39:35.570 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 02:39:35 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1[35569]: 2024-06-18T02:39:35.437+0000 7feaa0cbe700 -1 osd.1 0 waiting for initial osdmap 2024-06-18T02:39:35.570 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 02:39:35 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1[35569]: 2024-06-18T02:39:35.440+0000 7fea99651700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T02:39:35.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:35 smithi186 ceph-mon[25944]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:35.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:35 smithi186 ceph-mon[25944]: from='osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-18T02:39:35.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:35 smithi186 ceph-mon[25944]: osdmap e11: 2 total, 1 up, 2 in 2024-06-18T02:39:35.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:35 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:35.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:35 smithi186 ceph-mon[25944]: from='osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T02:39:36.042 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-06-18T02:39:36.042 INFO:teuthology.orchestra.run.smithi163.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-06-18T02:39:36.042 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10+0 records in 2024-06-18T02:39:36.043 INFO:teuthology.orchestra.run.smithi163.stderr:10+0 records out 2024-06-18T02:39:36.043 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125037 s, 839 MB/s 2024-06-18T02:39:36.043 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping successful for: 2024-06-18T02:39:36.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:36 smithi163 ceph-mon[29791]: from='osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2024-06-18T02:39:36.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:36 smithi163 ceph-mon[29791]: osdmap e12: 2 total, 1 up, 2 in 2024-06-18T02:39:36.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:36 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:36 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:36 smithi163 ceph-mon[25094]: from='osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2024-06-18T02:39:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:36 smithi163 ceph-mon[25094]: osdmap e12: 2 total, 1 up, 2 in 2024-06-18T02:39:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:36 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:36 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:36.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:36 smithi186 ceph-mon[25944]: from='osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2024-06-18T02:39:36.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:36 smithi186 ceph-mon[25944]: osdmap e12: 2 total, 1 up, 2 in 2024-06-18T02:39:36.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:36 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:36 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:36.916 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi163:vg_nvme/lv_2 2024-06-18T02:39:37.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:37 smithi163 ceph-mon[29791]: purged_snaps scrub starts 2024-06-18T02:39:37.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:37 smithi163 ceph-mon[29791]: purged_snaps scrub ok 2024-06-18T02:39:37.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:37 smithi163 ceph-mon[29791]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:37.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:37 smithi163 ceph-mon[29791]: osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684] boot 2024-06-18T02:39:37.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:37 smithi163 ceph-mon[29791]: osdmap e13: 2 total, 2 up, 2 in 2024-06-18T02:39:37.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:37 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:37.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:37 smithi163 ceph-mon[25094]: purged_snaps scrub starts 2024-06-18T02:39:37.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:37 smithi163 ceph-mon[25094]: purged_snaps scrub ok 2024-06-18T02:39:37.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:37 smithi163 ceph-mon[25094]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:37.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:37 smithi163 ceph-mon[25094]: osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684] boot 2024-06-18T02:39:37.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:37 smithi163 ceph-mon[25094]: osdmap e13: 2 total, 2 up, 2 in 2024-06-18T02:39:37.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:37 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:37.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:37 smithi186 ceph-mon[25944]: purged_snaps scrub starts 2024-06-18T02:39:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:37 smithi186 ceph-mon[25944]: purged_snaps scrub ok 2024-06-18T02:39:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:37 smithi186 ceph-mon[25944]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-18T02:39:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:37 smithi186 ceph-mon[25944]: osd.1 [v2:172.21.15.163:6810/1462975684,v1:172.21.15.163:6811/1462975684] boot 2024-06-18T02:39:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:37 smithi186 ceph-mon[25944]: osdmap e13: 2 total, 2 up, 2 in 2024-06-18T02:39:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:37 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:39:38.654 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:38 smithi163 ceph-mon[29791]: osdmap e14: 2 total, 2 up, 2 in 2024-06-18T02:39:38.654 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:38 smithi163 ceph-mon[25094]: osdmap e14: 2 total, 2 up, 2 in 2024-06-18T02:39:38.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:38 smithi186 ceph-mon[25944]: osdmap e14: 2 total, 2 up, 2 in 2024-06-18T02:39:39.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:39 smithi163 ceph-mon[25094]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:39.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:39 smithi163 ceph-mon[25094]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:39:39.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:39 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:39:39.511 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:39 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:39:39.511 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:39 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:39.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:39 smithi163 ceph-mon[29791]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:39.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:39 smithi163 ceph-mon[29791]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:39:39.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:39 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:39:39.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:39 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:39:39.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:39 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:39.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:39 smithi186 ceph-mon[25944]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:39.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:39 smithi186 ceph-mon[25944]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:39:39.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:39 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:39:39.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:39 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:39:39.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:39 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:41.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[29791]: Detected new or changed devices on smithi163 2024-06-18T02:39:41.022 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:41.022 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:39:41.022 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:39:41.022 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[29791]: Adjusting osd_memory_target on smithi163 to 8032M 2024-06-18T02:39:41.022 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:41.022 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:41.022 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:41.022 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:41.022 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:41.022 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:41.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[25094]: Detected new or changed devices on smithi163 2024-06-18T02:39:41.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:41.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:39:41.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:39:41.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[25094]: Adjusting osd_memory_target on smithi163 to 8032M 2024-06-18T02:39:41.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:41.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:41.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:41.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:41.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:41.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:40 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:40 smithi186 ceph-mon[25944]: Detected new or changed devices on smithi163 2024-06-18T02:39:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:40 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:40 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:39:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:40 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:39:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:40 smithi186 ceph-mon[25944]: Adjusting osd_memory_target on smithi163 to 8032M 2024-06-18T02:39:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:40 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:40 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:40 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:40 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:40 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:40 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:42.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:42 smithi163 ceph-mon[25094]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:42 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2640788020' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad504a3c-fa5c-4b4c-a079-a66b52088d68"}]: dispatch 2024-06-18T02:39:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:42 smithi163 ceph-mon[25094]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad504a3c-fa5c-4b4c-a079-a66b52088d68"}]: dispatch 2024-06-18T02:39:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:42 smithi163 ceph-mon[25094]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad504a3c-fa5c-4b4c-a079-a66b52088d68"}]': finished 2024-06-18T02:39:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:42 smithi163 ceph-mon[25094]: osdmap e15: 3 total, 2 up, 3 in 2024-06-18T02:39:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:42 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:39:42.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:42 smithi163 ceph-mon[29791]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:42.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:42 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2640788020' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad504a3c-fa5c-4b4c-a079-a66b52088d68"}]: dispatch 2024-06-18T02:39:42.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:42 smithi163 ceph-mon[29791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad504a3c-fa5c-4b4c-a079-a66b52088d68"}]: dispatch 2024-06-18T02:39:42.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:42 smithi163 ceph-mon[29791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad504a3c-fa5c-4b4c-a079-a66b52088d68"}]': finished 2024-06-18T02:39:42.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:42 smithi163 ceph-mon[29791]: osdmap e15: 3 total, 2 up, 3 in 2024-06-18T02:39:42.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:42 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:39:42.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:42 smithi186 ceph-mon[25944]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:42.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:42 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2640788020' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad504a3c-fa5c-4b4c-a079-a66b52088d68"}]: dispatch 2024-06-18T02:39:42.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:42 smithi186 ceph-mon[25944]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad504a3c-fa5c-4b4c-a079-a66b52088d68"}]: dispatch 2024-06-18T02:39:42.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:42 smithi186 ceph-mon[25944]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad504a3c-fa5c-4b4c-a079-a66b52088d68"}]': finished 2024-06-18T02:39:42.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:42 smithi186 ceph-mon[25944]: osdmap e15: 3 total, 2 up, 3 in 2024-06-18T02:39:42.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:42 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:39:43.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:43 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3315114412' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:39:43.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:43 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3315114412' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:39:43.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:43 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3315114412' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:39:44.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:44 smithi186 ceph-mon[25944]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:44.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:44 smithi163 ceph-mon[25094]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:44.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:44 smithi163 ceph-mon[29791]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:46.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:46 smithi163 ceph-mon[25094]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:46.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:46 smithi163 ceph-mon[29791]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:46.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:46 smithi186 ceph-mon[25944]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:48.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:48 smithi163 ceph-mon[29791]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:48.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:48 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-18T02:39:48.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:48 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:48.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:48 smithi163 ceph-mon[25094]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:48.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:48 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-18T02:39:48.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:48 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:48.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:48 smithi186 ceph-mon[25944]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:48.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:48 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-18T02:39:48.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:48 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:49.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:49 smithi186 ceph-mon[25944]: Deploying daemon osd.2 on smithi163 2024-06-18T02:39:49.380 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:49 smithi163 ceph-mon[29791]: Deploying daemon osd.2 on smithi163 2024-06-18T02:39:49.380 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:49 smithi163 ceph-mon[25094]: Deploying daemon osd.2 on smithi163 2024-06-18T02:39:50.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:50 smithi186 ceph-mon[25944]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:50.380 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:50 smithi163 ceph-mon[29791]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:50.380 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:50 smithi163 ceph-mon[25094]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:52.161 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:52 smithi163 ceph-mon[29791]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:52.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:52 smithi163 ceph-mon[25094]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:52.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:52 smithi186 ceph-mon[25944]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:54.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:53 smithi163 ceph-mon[29791]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:54.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:53 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:54.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:53 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:54.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:53 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:54.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:53 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:54.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:53 smithi163 ceph-mon[25094]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:54.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:53 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:54.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:53 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:54.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:53 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:54.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:53 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:54.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:53 smithi186 ceph-mon[25944]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:54.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:53 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:54.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:53 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:54.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:53 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:54.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:53 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:55.271 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:54 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:39:55.271 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:54 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:39:55.271 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:54 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:39:55.271 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:54 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:39:55.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:54 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:39:55.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:54 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:39:56.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:56 smithi163 ceph-mon[29791]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:56.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:56 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:56.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:56 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:56.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:56 smithi163 ceph-mon[25094]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:56.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:56 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:56.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:56 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:56.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:56 smithi186 ceph-mon[25944]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:56.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:56 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:56.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:56 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:56.488 INFO:teuthology.orchestra.run.smithi163.stdout:Created osd(s) 2 on host 'smithi163' 2024-06-18T02:39:57.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:57 smithi163 ceph-mon[25094]: from='osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-18T02:39:57.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:57.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:57.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:57.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:57.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:57 smithi163 ceph-mon[29791]: from='osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-18T02:39:57.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:57 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:57.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:57 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:57.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:57 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:57.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:57 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:57.062 DEBUG:teuthology.orchestra.run.smithi163:osd.2> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.2.service 2024-06-18T02:39:57.064 INFO:tasks.cephadm:Deploying osd.3 on smithi163 with /dev/vg_nvme/lv_1... 2024-06-18T02:39:57.065 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-06-18T02:39:57.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:57 smithi186 ceph-mon[25944]: from='osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-18T02:39:57.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:57 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:39:57.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:57 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:39:57.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:57 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:39:57.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:57 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:39:58.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:58 smithi163 ceph-mon[25094]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:58.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:58 smithi163 ceph-mon[25094]: from='osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-18T02:39:58.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:58 smithi163 ceph-mon[25094]: osdmap e16: 3 total, 2 up, 3 in 2024-06-18T02:39:58.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:58 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:39:58.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:58 smithi163 ceph-mon[25094]: from='osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T02:39:58.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:58 smithi163 ceph-mon[29791]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:58.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:58 smithi163 ceph-mon[29791]: from='osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-18T02:39:58.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:58 smithi163 ceph-mon[29791]: osdmap e16: 3 total, 2 up, 3 in 2024-06-18T02:39:58.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:58 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:39:58.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:58 smithi163 ceph-mon[29791]: from='osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T02:39:58.293 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 02:39:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2[38473]: 2024-06-18T02:39:58.055+0000 7f7f15459700 -1 osd.2 0 waiting for initial osdmap 2024-06-18T02:39:58.293 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 02:39:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2[38473]: 2024-06-18T02:39:58.059+0000 7f7f0e5ed700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T02:39:58.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:58 smithi186 ceph-mon[25944]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:39:58.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:58 smithi186 ceph-mon[25944]: from='osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-18T02:39:58.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:58 smithi186 ceph-mon[25944]: osdmap e16: 3 total, 2 up, 3 in 2024-06-18T02:39:58.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:58 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:39:58.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:58 smithi186 ceph-mon[25944]: from='osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T02:39:59.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:59 smithi163 ceph-mon[25094]: from='osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2024-06-18T02:39:59.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:59 smithi163 ceph-mon[25094]: osdmap e17: 3 total, 2 up, 3 in 2024-06-18T02:39:59.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:59 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:39:59.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:39:59 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:39:59.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:59 smithi163 ceph-mon[29791]: from='osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2024-06-18T02:39:59.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:59 smithi163 ceph-mon[29791]: osdmap e17: 3 total, 2 up, 3 in 2024-06-18T02:39:59.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:59 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:39:59.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:39:59 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:39:59.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:59 smithi186 ceph-mon[25944]: from='osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2024-06-18T02:39:59.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:59 smithi186 ceph-mon[25944]: osdmap e17: 3 total, 2 up, 3 in 2024-06-18T02:39:59.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:59 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:39:59.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:39:59 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:40:00.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[25094]: purged_snaps scrub starts 2024-06-18T02:40:00.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[25094]: purged_snaps scrub ok 2024-06-18T02:40:00.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[25094]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:40:00.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[25094]: osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700] boot 2024-06-18T02:40:00.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[25094]: osdmap e18: 3 total, 3 up, 3 in 2024-06-18T02:40:00.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:40:00.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[25094]: overall HEALTH_OK 2024-06-18T02:40:00.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[29791]: purged_snaps scrub starts 2024-06-18T02:40:00.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[29791]: purged_snaps scrub ok 2024-06-18T02:40:00.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[29791]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:40:00.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[29791]: osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700] boot 2024-06-18T02:40:00.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[29791]: osdmap e18: 3 total, 3 up, 3 in 2024-06-18T02:40:00.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:40:00.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:00 smithi163 ceph-mon[29791]: overall HEALTH_OK 2024-06-18T02:40:00.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:00 smithi186 ceph-mon[25944]: purged_snaps scrub starts 2024-06-18T02:40:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:00 smithi186 ceph-mon[25944]: purged_snaps scrub ok 2024-06-18T02:40:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:00 smithi186 ceph-mon[25944]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-18T02:40:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:00 smithi186 ceph-mon[25944]: osd.2 [v2:172.21.15.163:6818/923602700,v1:172.21.15.163:6819/923602700] boot 2024-06-18T02:40:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:00 smithi186 ceph-mon[25944]: osdmap e18: 3 total, 3 up, 3 in 2024-06-18T02:40:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:00 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:40:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:00 smithi186 ceph-mon[25944]: overall HEALTH_OK 2024-06-18T02:40:01.041 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-06-18T02:40:01.042 INFO:teuthology.orchestra.run.smithi163.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-06-18T02:40:01.042 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10+0 records in 2024-06-18T02:40:01.042 INFO:teuthology.orchestra.run.smithi163.stderr:10+0 records out 2024-06-18T02:40:01.042 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0149859 s, 700 MB/s 2024-06-18T02:40:01.042 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping successful for: 2024-06-18T02:40:01.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[25094]: osdmap e19: 3 total, 3 up, 3 in 2024-06-18T02:40:01.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-06-18T02:40:01.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:01.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:01.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:01.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:01.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:01.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:01.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[29791]: osdmap e19: 3 total, 3 up, 3 in 2024-06-18T02:40:01.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-06-18T02:40:01.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:01.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:01.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:01.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:01.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:01.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:01 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:01.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:01 smithi186 ceph-mon[25944]: osdmap e19: 3 total, 3 up, 3 in 2024-06-18T02:40:01.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:01 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-06-18T02:40:01.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:01 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:01.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:01 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:01.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:01 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:01.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:01 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:01.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:01 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:01.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:01 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:01.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi163:vg_nvme/lv_1 2024-06-18T02:40:02.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:02 smithi163 ceph-mon[25094]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:02.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:02 smithi163 ceph-mon[25094]: Detected new or changed devices on smithi163 2024-06-18T02:40:02.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:02 smithi163 ceph-mon[25094]: Adjusting osd_memory_target on smithi163 to 5354M 2024-06-18T02:40:02.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:02 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-06-18T02:40:02.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:02 smithi163 ceph-mon[25094]: osdmap e20: 3 total, 3 up, 3 in 2024-06-18T02:40:02.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:02 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-06-18T02:40:02.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:02 smithi163 ceph-mon[29791]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:02.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:02 smithi163 ceph-mon[29791]: Detected new or changed devices on smithi163 2024-06-18T02:40:02.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:02 smithi163 ceph-mon[29791]: Adjusting osd_memory_target on smithi163 to 5354M 2024-06-18T02:40:02.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:02 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-06-18T02:40:02.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:02 smithi163 ceph-mon[29791]: osdmap e20: 3 total, 3 up, 3 in 2024-06-18T02:40:02.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:02 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-06-18T02:40:02.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:02 smithi186 ceph-mon[25944]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:02 smithi186 ceph-mon[25944]: Detected new or changed devices on smithi163 2024-06-18T02:40:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:02 smithi186 ceph-mon[25944]: Adjusting osd_memory_target on smithi163 to 5354M 2024-06-18T02:40:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:02 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-06-18T02:40:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:02 smithi186 ceph-mon[25944]: osdmap e20: 3 total, 3 up, 3 in 2024-06-18T02:40:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:02 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-06-18T02:40:03.433 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:03 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-06-18T02:40:03.433 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:03 smithi163 ceph-mon[29791]: osdmap e21: 3 total, 3 up, 3 in 2024-06-18T02:40:03.433 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:03 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-06-18T02:40:03.433 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:03 smithi163 ceph-mon[25094]: osdmap e21: 3 total, 3 up, 3 in 2024-06-18T02:40:03.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:03 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-06-18T02:40:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:03 smithi186 ceph-mon[25944]: osdmap e21: 3 total, 3 up, 3 in 2024-06-18T02:40:03.694 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 02:40:03 smithi163 sudo[39803]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-18T02:40:03.695 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 02:40:03 smithi163 sudo[39803]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-18T02:40:03.695 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 02:40:03 smithi163 sudo[39803]: pam_unix(sudo:session): session closed for user root 2024-06-18T02:40:03.955 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 02:40:03 smithi163 sudo[39843]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-18T02:40:03.956 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 02:40:03 smithi163 sudo[39843]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-18T02:40:03.956 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 02:40:03 smithi163 sudo[39843]: pam_unix(sudo:session): session closed for user root 2024-06-18T02:40:04.226 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 02:40:03 smithi163 sudo[39886]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-18T02:40:04.226 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 02:40:03 smithi163 sudo[39886]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-18T02:40:04.226 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 02:40:03 smithi163 sudo[39886]: pam_unix(sudo:session): session closed for user root 2024-06-18T02:40:04.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:04 smithi163 ceph-mon[29791]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:04.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:04 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:40:04.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:04 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:40:04.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:04 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:04.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:04 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:04.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:04 smithi163 ceph-mon[25094]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:04.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:04 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:40:04.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:04 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:40:04.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:04 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:04.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:04 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:04.484 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 02:40:04 smithi163 sudo[39901]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-18T02:40:04.484 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 02:40:04 smithi163 sudo[39901]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-18T02:40:04.484 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 02:40:04 smithi163 sudo[39901]: pam_unix(sudo:session): session closed for user root 2024-06-18T02:40:04.484 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 02:40:04 smithi163 sudo[39908]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-18T02:40:04.484 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 02:40:04 smithi163 sudo[39908]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-18T02:40:04.484 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 02:40:04 smithi163 sudo[39908]: pam_unix(sudo:session): session closed for user root 2024-06-18T02:40:04.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:04 smithi186 ceph-mon[25944]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:04 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:40:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:04 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:40:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:04 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:04 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:04.792 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 02:40:04 smithi163 sudo[39915]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-18T02:40:04.792 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 02:40:04 smithi163 sudo[39915]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-18T02:40:04.792 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 02:40:04 smithi163 sudo[39915]: pam_unix(sudo:session): session closed for user root 2024-06-18T02:40:05.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:04 smithi163 sudo[39937]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-18T02:40:05.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:04 smithi163 sudo[39937]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-18T02:40:05.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:04 smithi163 sudo[39937]: pam_unix(sudo:session): session closed for user root 2024-06-18T02:40:05.496 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:05 smithi163 sudo[39946]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-18T02:40:05.496 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:05 smithi163 sudo[39946]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-18T02:40:05.496 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:05 smithi163 sudo[39946]: pam_unix(sudo:session): session closed for user root 2024-06-18T02:40:05.496 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:05 smithi163 ceph-mon[25094]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:40:05.496 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:05 smithi163 ceph-mon[25094]: osdmap e22: 3 total, 3 up, 3 in 2024-06-18T02:40:05.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:05 smithi186 ceph-mon[25944]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:40:05.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:05 smithi186 ceph-mon[25944]: osdmap e22: 3 total, 3 up, 3 in 2024-06-18T02:40:05.760 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:05 smithi163 sudo[39956]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-18T02:40:05.760 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:05 smithi163 sudo[39956]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-18T02:40:05.761 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:05 smithi163 sudo[39956]: pam_unix(sudo:session): session closed for user root 2024-06-18T02:40:06.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:05 smithi163 sudo[39986]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-18T02:40:06.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:05 smithi163 sudo[39986]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-18T02:40:06.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:05 smithi163 sudo[39986]: pam_unix(sudo:session): session closed for user root 2024-06-18T02:40:06.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:05 smithi163 ceph-mon[29791]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:40:06.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:05 smithi163 ceph-mon[29791]: osdmap e22: 3 total, 3 up, 3 in 2024-06-18T02:40:06.255 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 sudo[27448]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-18T02:40:06.255 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 sudo[27448]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-18T02:40:06.255 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 sudo[27448]: pam_unix(sudo:session): session closed for user root 2024-06-18T02:40:06.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: pgmap v77: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:06.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-18T02:40:06.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-18T02:40:06.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:40:06.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:40:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:40:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-18T02:40:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:40:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: mgrmap e15: y(active, since 2m), standbys: x 2024-06-18T02:40:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:40:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:40:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-18T02:40:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:40:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:40:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:40:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1207054337' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9f9da4f-4afc-48cf-92a2-f714ca3f4c4f"}]: dispatch 2024-06-18T02:40:06.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: pgmap v77: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: mgrmap e15: y(active, since 2m), standbys: x 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:40:06.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:40:06.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:06 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1207054337' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9f9da4f-4afc-48cf-92a2-f714ca3f4c4f"}]: dispatch 2024-06-18T02:40:06.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 sudo[27451]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-18T02:40:06.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 sudo[27451]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-18T02:40:06.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 sudo[27451]: pam_unix(sudo:session): session closed for user root 2024-06-18T02:40:06.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: pgmap v77: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:06.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: mgrmap e15: y(active, since 2m), standbys: x 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:40:06.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:06 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1207054337' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9f9da4f-4afc-48cf-92a2-f714ca3f4c4f"}]: dispatch 2024-06-18T02:40:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[25094]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-18T02:40:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[25094]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-18T02:40:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1207054337' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9f9da4f-4afc-48cf-92a2-f714ca3f4c4f"}]': finished 2024-06-18T02:40:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[25094]: osdmap e23: 4 total, 3 up, 4 in 2024-06-18T02:40:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[25094]: pgmap v79: 1 pgs: 1 creating+activating; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:07.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3472793408' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:40:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[29791]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-18T02:40:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[29791]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-18T02:40:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1207054337' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9f9da4f-4afc-48cf-92a2-f714ca3f4c4f"}]': finished 2024-06-18T02:40:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[29791]: osdmap e23: 4 total, 3 up, 4 in 2024-06-18T02:40:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[29791]: pgmap v79: 1 pgs: 1 creating+activating; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:07 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3472793408' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:40:07.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:07 smithi186 ceph-mon[25944]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-18T02:40:07.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:07 smithi186 ceph-mon[25944]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-18T02:40:07.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:07 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1207054337' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9f9da4f-4afc-48cf-92a2-f714ca3f4c4f"}]': finished 2024-06-18T02:40:07.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:07 smithi186 ceph-mon[25944]: osdmap e23: 4 total, 3 up, 4 in 2024-06-18T02:40:07.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:07 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:07.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:07 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:07.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:07 smithi186 ceph-mon[25944]: pgmap v79: 1 pgs: 1 creating+activating; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:07.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:07 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3472793408' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:40:09.612 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:09 smithi163 ceph-mon[29791]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:09.612 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:09 smithi163 ceph-mon[25094]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:09.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:09 smithi186 ceph-mon[25944]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:11.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:11 smithi163 ceph-mon[29791]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:11.346 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:11 smithi163 ceph-mon[25094]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:11.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:11 smithi186 ceph-mon[25944]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:12.760 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:12 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-18T02:40:12.760 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:12 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:12.760 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:12 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-18T02:40:12.760 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:12 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:12.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:12 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-18T02:40:12.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:12 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:13.782 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:13 smithi163 ceph-mon[29791]: Deploying daemon osd.3 on smithi163 2024-06-18T02:40:13.782 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:13 smithi163 ceph-mon[29791]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:13.783 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:13 smithi163 ceph-mon[25094]: Deploying daemon osd.3 on smithi163 2024-06-18T02:40:13.783 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:13 smithi163 ceph-mon[25094]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:13.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:13 smithi186 ceph-mon[25944]: Deploying daemon osd.3 on smithi163 2024-06-18T02:40:13.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:13 smithi186 ceph-mon[25944]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:15.716 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:15 smithi163 ceph-mon[29791]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:15.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:15 smithi163 ceph-mon[25094]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:15.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:15 smithi186 ceph-mon[25944]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:17.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:17 smithi163 ceph-mon[25094]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:17.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:17 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:17.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:17 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:40:17.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:17 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:17.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:17 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:40:17.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:17 smithi163 ceph-mon[29791]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:17.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:17 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:17.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:17 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:40:17.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:17 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:17.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:17 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:40:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:17 smithi186 ceph-mon[25944]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:17 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:17 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:40:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:17 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:17 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:40:19.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:19 smithi163 ceph-mon[29791]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:19.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:19 smithi163 ceph-mon[25094]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:19.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:19 smithi186 ceph-mon[25944]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:20.704 INFO:teuthology.orchestra.run.smithi163.stdout:Created osd(s) 3 on host 'smithi163' 2024-06-18T02:40:21.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:21.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:21.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[29791]: from='osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-18T02:40:21.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:21.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:40:21.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:21.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:40:21.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:21.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:21.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[25094]: from='osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-18T02:40:21.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:21.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:40:21.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:21.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:20 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:40:21.303 DEBUG:teuthology.orchestra.run.smithi163:osd.3> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.3.service 2024-06-18T02:40:21.306 INFO:tasks.cephadm:Deploying osd.4 on smithi186 with /dev/vg_nvme/lv_4... 2024-06-18T02:40:21.306 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-06-18T02:40:21.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:20 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:20 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:20 smithi186 ceph-mon[25944]: from='osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-18T02:40:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:20 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:20 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:40:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:20 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:20 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:40:22.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:21 smithi163 ceph-mon[25094]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:22.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:21 smithi163 ceph-mon[25094]: from='osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-18T02:40:22.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:21 smithi163 ceph-mon[25094]: osdmap e24: 4 total, 3 up, 4 in 2024-06-18T02:40:22.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:21 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:22.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:21 smithi163 ceph-mon[25094]: from='osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T02:40:22.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:21 smithi163 ceph-mon[29791]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:22.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:21 smithi163 ceph-mon[29791]: from='osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-18T02:40:22.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:21 smithi163 ceph-mon[29791]: osdmap e24: 4 total, 3 up, 4 in 2024-06-18T02:40:22.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:21 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:22.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:21 smithi163 ceph-mon[29791]: from='osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T02:40:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:21 smithi186 ceph-mon[25944]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:21 smithi186 ceph-mon[25944]: from='osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-18T02:40:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:21 smithi186 ceph-mon[25944]: osdmap e24: 4 total, 3 up, 4 in 2024-06-18T02:40:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:21 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:21 smithi186 ceph-mon[25944]: from='osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T02:40:22.645 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 02:40:22 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3[41430]: 2024-06-18T02:40:22.367+0000 7f94e00bc700 -1 osd.3 0 waiting for initial osdmap 2024-06-18T02:40:22.645 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 02:40:22 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3[41430]: 2024-06-18T02:40:22.371+0000 7f94dca57700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T02:40:23.291 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-06-18T02:40:23.291 INFO:teuthology.orchestra.run.smithi186.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-06-18T02:40:23.292 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10+0 records in 2024-06-18T02:40:23.292 INFO:teuthology.orchestra.run.smithi186.stderr:10+0 records out 2024-06-18T02:40:23.292 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113455 s, 924 MB/s 2024-06-18T02:40:23.292 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping successful for: 2024-06-18T02:40:23.562 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:23 smithi186 ceph-mon[25944]: from='osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2024-06-18T02:40:23.562 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:23 smithi186 ceph-mon[25944]: osdmap e25: 4 total, 3 up, 4 in 2024-06-18T02:40:23.562 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:23 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:23.562 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:23 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:23.563 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:23 smithi186 ceph-mon[25944]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:23.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:23 smithi163 ceph-mon[29791]: from='osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2024-06-18T02:40:23.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:23 smithi163 ceph-mon[29791]: osdmap e25: 4 total, 3 up, 4 in 2024-06-18T02:40:23.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:23 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:23.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:23 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:23.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:23 smithi163 ceph-mon[29791]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:23 smithi163 ceph-mon[25094]: from='osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2024-06-18T02:40:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:23 smithi163 ceph-mon[25094]: osdmap e25: 4 total, 3 up, 4 in 2024-06-18T02:40:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:23 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:23 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:23 smithi163 ceph-mon[25094]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-18T02:40:23.941 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi186:vg_nvme/lv_4 2024-06-18T02:40:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: purged_snaps scrub starts 2024-06-18T02:40:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: purged_snaps scrub ok 2024-06-18T02:40:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135] boot 2024-06-18T02:40:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: osdmap e26: 4 total, 4 up, 4 in 2024-06-18T02:40:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: Detected new or changed devices on smithi163 2024-06-18T02:40:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:24.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:24.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:24.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:24.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: Adjusting osd_memory_target on smithi163 to 4016M 2024-06-18T02:40:24.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:24.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:24 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:24.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: purged_snaps scrub starts 2024-06-18T02:40:24.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: purged_snaps scrub ok 2024-06-18T02:40:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135] boot 2024-06-18T02:40:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: osdmap e26: 4 total, 4 up, 4 in 2024-06-18T02:40:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: Detected new or changed devices on smithi163 2024-06-18T02:40:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: Adjusting osd_memory_target on smithi163 to 4016M 2024-06-18T02:40:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:24.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: purged_snaps scrub starts 2024-06-18T02:40:24.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: purged_snaps scrub ok 2024-06-18T02:40:24.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: osd.3 [v2:172.21.15.163:6826/2061575135,v1:172.21.15.163:6827/2061575135] boot 2024-06-18T02:40:24.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: osdmap e26: 4 total, 4 up, 4 in 2024-06-18T02:40:24.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:40:24.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: Detected new or changed devices on smithi163 2024-06-18T02:40:24.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:24.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:24.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:24.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:24.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:24.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: Adjusting osd_memory_target on smithi163 to 4016M 2024-06-18T02:40:24.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:24.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:24 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:25.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:25 smithi163 ceph-mon[25094]: osdmap e27: 4 total, 4 up, 4 in 2024-06-18T02:40:25.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:25 smithi163 ceph-mon[25094]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:25.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:25 smithi163 ceph-mon[29791]: osdmap e27: 4 total, 4 up, 4 in 2024-06-18T02:40:25.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:25 smithi163 ceph-mon[29791]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:25.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:25 smithi186 ceph-mon[25944]: osdmap e27: 4 total, 4 up, 4 in 2024-06-18T02:40:25.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:25 smithi186 ceph-mon[25944]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:26.464 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:26 smithi186 ceph-mon[25944]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:40:26.464 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:26 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:40:26.464 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:26 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:40:26.465 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:26 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:26 smithi163 ceph-mon[25094]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:40:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:26 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:40:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:26 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:40:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:26 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:26 smithi163 ceph-mon[29791]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:40:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:26 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:40:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:26 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:40:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:26 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:27.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:27 smithi163 ceph-mon[25094]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:27.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:27 smithi163 ceph-mon[29791]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:27.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:27 smithi186 ceph-mon[25944]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:28.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:28 smithi163 ceph-mon[25094]: from='client.? 172.21.15.186:0/1982811488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12a9d06c-9aab-4659-8533-f57336078d0c"}]: dispatch 2024-06-18T02:40:28.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:28 smithi163 ceph-mon[25094]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12a9d06c-9aab-4659-8533-f57336078d0c"}]: dispatch 2024-06-18T02:40:28.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:28 smithi163 ceph-mon[25094]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12a9d06c-9aab-4659-8533-f57336078d0c"}]': finished 2024-06-18T02:40:28.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:28 smithi163 ceph-mon[25094]: osdmap e28: 5 total, 4 up, 5 in 2024-06-18T02:40:28.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:28 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:28.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:28 smithi163 ceph-mon[29791]: from='client.? 172.21.15.186:0/1982811488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12a9d06c-9aab-4659-8533-f57336078d0c"}]: dispatch 2024-06-18T02:40:28.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:28 smithi163 ceph-mon[29791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12a9d06c-9aab-4659-8533-f57336078d0c"}]: dispatch 2024-06-18T02:40:28.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:28 smithi163 ceph-mon[29791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12a9d06c-9aab-4659-8533-f57336078d0c"}]': finished 2024-06-18T02:40:28.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:28 smithi163 ceph-mon[29791]: osdmap e28: 5 total, 4 up, 5 in 2024-06-18T02:40:28.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:28 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:28 smithi186 ceph-mon[25944]: from='client.? 172.21.15.186:0/1982811488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12a9d06c-9aab-4659-8533-f57336078d0c"}]: dispatch 2024-06-18T02:40:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:28 smithi186 ceph-mon[25944]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12a9d06c-9aab-4659-8533-f57336078d0c"}]: dispatch 2024-06-18T02:40:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:28 smithi186 ceph-mon[25944]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12a9d06c-9aab-4659-8533-f57336078d0c"}]': finished 2024-06-18T02:40:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:28 smithi186 ceph-mon[25944]: osdmap e28: 5 total, 4 up, 5 in 2024-06-18T02:40:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:28 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:29 smithi163 ceph-mon[25094]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:29 smithi163 ceph-mon[25094]: from='client.? 172.21.15.186:0/2409088713' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:40:29.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:29 smithi163 ceph-mon[29791]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:29.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:29 smithi163 ceph-mon[29791]: from='client.? 172.21.15.186:0/2409088713' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:40:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:29 smithi186 ceph-mon[25944]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:29 smithi186 ceph-mon[25944]: from='client.? 172.21.15.186:0/2409088713' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:40:31.436 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:31 smithi186 ceph-mon[25944]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:31.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:31 smithi163 ceph-mon[25094]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:31.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:31 smithi163 ceph-mon[29791]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:33.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:33 smithi186 ceph-mon[25944]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:33.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:33 smithi163 ceph-mon[25094]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:33.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:33 smithi163 ceph-mon[29791]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:34.685 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:34 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-18T02:40:34.685 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:34 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:34.685 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:34 smithi186 ceph-mon[25944]: Deploying daemon osd.4 on smithi186 2024-06-18T02:40:34.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:34 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-18T02:40:34.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:34 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:34.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:34 smithi163 ceph-mon[25094]: Deploying daemon osd.4 on smithi186 2024-06-18T02:40:34.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:34 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-18T02:40:34.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:34 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:34.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:34 smithi163 ceph-mon[29791]: Deploying daemon osd.4 on smithi186 2024-06-18T02:40:35.603 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:35 smithi186 ceph-mon[25944]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:35.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:35 smithi163 ceph-mon[29791]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:35.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:35 smithi163 ceph-mon[25094]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:37.713 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:37 smithi186 ceph-mon[25944]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:37.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:37 smithi163 ceph-mon[25094]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:37.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:37 smithi163 ceph-mon[29791]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:40.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:39 smithi163 ceph-mon[29791]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:40.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:39 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:40.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:39 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:40:40.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:39 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:40.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:39 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:40:40.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:39 smithi163 ceph-mon[25094]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:40.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:39 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:40.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:39 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:40:40.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:39 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:40.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:39 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:40:40.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:39 smithi186 ceph-mon[25944]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:40.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:39 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:40.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:39 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:40:40.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:39 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:40.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:39 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:40:41.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:41 smithi163 ceph-mon[25094]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:41.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:41 smithi163 ceph-mon[29791]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:41.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:41 smithi186 ceph-mon[25944]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:41.708 INFO:teuthology.orchestra.run.smithi186.stdout:Created osd(s) 4 on host 'smithi186' 2024-06-18T02:40:42.345 DEBUG:teuthology.orchestra.run.smithi186:osd.4> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.4.service 2024-06-18T02:40:42.348 INFO:tasks.cephadm:Deploying osd.5 on smithi186 with /dev/vg_nvme/lv_3... 2024-06-18T02:40:42.348 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-06-18T02:40:43.021 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:42 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:43.021 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:42 smithi186 ceph-mon[25944]: from='osd.4 [v2:172.21.15.186:6800/368766852,v1:172.21.15.186:6801/368766852]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-18T02:40:43.021 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:42 smithi186 ceph-mon[25944]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-18T02:40:43.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:42 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:43.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:42 smithi163 ceph-mon[29791]: from='osd.4 [v2:172.21.15.186:6800/368766852,v1:172.21.15.186:6801/368766852]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-18T02:40:43.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:42 smithi163 ceph-mon[29791]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-18T02:40:43.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:42 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:43.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:42 smithi163 ceph-mon[25094]: from='osd.4 [v2:172.21.15.186:6800/368766852,v1:172.21.15.186:6801/368766852]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-18T02:40:43.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:42 smithi163 ceph-mon[25094]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-18T02:40:44.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[29791]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:44.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[29791]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-18T02:40:44.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[29791]: osdmap e29: 5 total, 4 up, 5 in 2024-06-18T02:40:44.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:44.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[29791]: from='osd.4 [v2:172.21.15.186:6800/368766852,v1:172.21.15.186:6801/368766852]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:40:44.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[29791]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:40:44.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:44.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[25094]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:44.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[25094]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-18T02:40:44.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[25094]: osdmap e29: 5 total, 4 up, 5 in 2024-06-18T02:40:44.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:44.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[25094]: from='osd.4 [v2:172.21.15.186:6800/368766852,v1:172.21.15.186:6801/368766852]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:40:44.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[25094]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:40:44.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:43 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:43 smithi186 ceph-mon[25944]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:43 smithi186 ceph-mon[25944]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-18T02:40:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:43 smithi186 ceph-mon[25944]: osdmap e29: 5 total, 4 up, 5 in 2024-06-18T02:40:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:43 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:43 smithi186 ceph-mon[25944]: from='osd.4 [v2:172.21.15.186:6800/368766852,v1:172.21.15.186:6801/368766852]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:40:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:43 smithi186 ceph-mon[25944]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:40:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:43 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:44.068 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 02:40:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[29287]: 2024-06-18T02:40:43.720+0000 7f3033a7b700 -1 osd.4 0 waiting for initial osdmap 2024-06-18T02:40:44.069 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 02:40:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[29287]: 2024-06-18T02:40:43.725+0000 7f302e412700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T02:40:44.590 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-06-18T02:40:44.590 INFO:teuthology.orchestra.run.smithi186.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-06-18T02:40:44.590 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10+0 records in 2024-06-18T02:40:44.590 INFO:teuthology.orchestra.run.smithi186.stderr:10+0 records out 2024-06-18T02:40:44.590 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124283 s, 844 MB/s 2024-06-18T02:40:44.590 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping successful for: 2024-06-18T02:40:45.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:44 smithi163 ceph-mon[29791]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-06-18T02:40:45.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:44 smithi163 ceph-mon[29791]: osdmap e30: 5 total, 4 up, 5 in 2024-06-18T02:40:45.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:44 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:45.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:44 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:44 smithi163 ceph-mon[25094]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-06-18T02:40:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:44 smithi163 ceph-mon[25094]: osdmap e30: 5 total, 4 up, 5 in 2024-06-18T02:40:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:44 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:44 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:45.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:44 smithi186 ceph-mon[25944]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-06-18T02:40:45.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:44 smithi186 ceph-mon[25944]: osdmap e30: 5 total, 4 up, 5 in 2024-06-18T02:40:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:44 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:44 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:45.206 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi186:vg_nvme/lv_3 2024-06-18T02:40:46.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[29791]: purged_snaps scrub starts 2024-06-18T02:40:46.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[29791]: purged_snaps scrub ok 2024-06-18T02:40:46.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[29791]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:46.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[29791]: osd.4 [v2:172.21.15.186:6800/368766852,v1:172.21.15.186:6801/368766852] boot 2024-06-18T02:40:46.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[29791]: osdmap e31: 5 total, 5 up, 5 in 2024-06-18T02:40:46.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:46.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[29791]: osdmap e32: 5 total, 5 up, 5 in 2024-06-18T02:40:46.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[25094]: purged_snaps scrub starts 2024-06-18T02:40:46.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[25094]: purged_snaps scrub ok 2024-06-18T02:40:46.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[25094]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:46.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[25094]: osd.4 [v2:172.21.15.186:6800/368766852,v1:172.21.15.186:6801/368766852] boot 2024-06-18T02:40:46.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[25094]: osdmap e31: 5 total, 5 up, 5 in 2024-06-18T02:40:46.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:46.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:45 smithi163 ceph-mon[25094]: osdmap e32: 5 total, 5 up, 5 in 2024-06-18T02:40:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:45 smithi186 ceph-mon[25944]: purged_snaps scrub starts 2024-06-18T02:40:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:45 smithi186 ceph-mon[25944]: purged_snaps scrub ok 2024-06-18T02:40:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:45 smithi186 ceph-mon[25944]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-18T02:40:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:45 smithi186 ceph-mon[25944]: osd.4 [v2:172.21.15.186:6800/368766852,v1:172.21.15.186:6801/368766852] boot 2024-06-18T02:40:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:45 smithi186 ceph-mon[25944]: osdmap e31: 5 total, 5 up, 5 in 2024-06-18T02:40:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:45 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:40:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:45 smithi186 ceph-mon[25944]: osdmap e32: 5 total, 5 up, 5 in 2024-06-18T02:40:47.516 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:47 smithi186 ceph-mon[25944]: osdmap e33: 5 total, 5 up, 5 in 2024-06-18T02:40:47.516 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:47 smithi186 ceph-mon[25944]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:40:47.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:47 smithi163 ceph-mon[29791]: osdmap e33: 5 total, 5 up, 5 in 2024-06-18T02:40:47.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:47 smithi163 ceph-mon[29791]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:40:47.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:47 smithi163 ceph-mon[25094]: osdmap e33: 5 total, 5 up, 5 in 2024-06-18T02:40:47.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:47 smithi163 ceph-mon[25094]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:40:48.433 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:48 smithi186 ceph-mon[25944]: osdmap e34: 5 total, 5 up, 5 in 2024-06-18T02:40:48.433 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:48 smithi186 ceph-mon[25944]: Detected new or changed devices on smithi186 2024-06-18T02:40:48.433 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:48 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:48.433 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:48 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:48.433 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:48 smithi186 ceph-mon[25944]: Adjusting osd_memory_target on smithi186 to 17088M 2024-06-18T02:40:48.433 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:48 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:48.433 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:48 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:48.433 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:48 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:40:48.434 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:48 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:48.434 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:48 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:40:48.434 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:48 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:48.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[29791]: osdmap e34: 5 total, 5 up, 5 in 2024-06-18T02:40:48.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[29791]: Detected new or changed devices on smithi186 2024-06-18T02:40:48.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:48.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:48.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[29791]: Adjusting osd_memory_target on smithi186 to 17088M 2024-06-18T02:40:48.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:48.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:48.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:40:48.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:48.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:40:48.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:48.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[25094]: osdmap e34: 5 total, 5 up, 5 in 2024-06-18T02:40:48.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[25094]: Detected new or changed devices on smithi186 2024-06-18T02:40:48.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:48.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:40:48.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[25094]: Adjusting osd_memory_target on smithi186 to 17088M 2024-06-18T02:40:48.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:48.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:48.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:40:48.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:48.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:40:48.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:48 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:40:49.450 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:49 smithi186 ceph-mon[25944]: osdmap e35: 5 total, 5 up, 5 in 2024-06-18T02:40:49.450 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:49 smithi186 ceph-mon[25944]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 141 KiB/s, 0 objects/s recovering 2024-06-18T02:40:49.450 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:49 smithi186 ceph-mon[25944]: from='client.24224 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:40:49.450 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:49 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:40:49.450 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:49 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:40:49.450 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:49 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:49.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:49 smithi163 ceph-mon[29791]: osdmap e35: 5 total, 5 up, 5 in 2024-06-18T02:40:49.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:49 smithi163 ceph-mon[29791]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 141 KiB/s, 0 objects/s recovering 2024-06-18T02:40:49.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:49 smithi163 ceph-mon[29791]: from='client.24224 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:40:49.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:49 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:40:49.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:49 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:40:49.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:49 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:49.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:49 smithi163 ceph-mon[25094]: osdmap e35: 5 total, 5 up, 5 in 2024-06-18T02:40:49.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:49 smithi163 ceph-mon[25094]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 141 KiB/s, 0 objects/s recovering 2024-06-18T02:40:49.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:49 smithi163 ceph-mon[25094]: from='client.24224 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:40:49.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:49 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:40:49.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:49 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:40:49.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:49 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:51.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:51 smithi163 ceph-mon[29791]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-06-18T02:40:51.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:51 smithi163 ceph-mon[29791]: from='client.? 172.21.15.186:0/3537997450' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9bb0e529-2a21-425f-9da9-aa81ae22a52c"}]: dispatch 2024-06-18T02:40:51.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:51 smithi163 ceph-mon[29791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9bb0e529-2a21-425f-9da9-aa81ae22a52c"}]: dispatch 2024-06-18T02:40:51.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:51 smithi163 ceph-mon[29791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9bb0e529-2a21-425f-9da9-aa81ae22a52c"}]': finished 2024-06-18T02:40:51.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:51 smithi163 ceph-mon[29791]: osdmap e36: 6 total, 5 up, 6 in 2024-06-18T02:40:51.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:51 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:40:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:51 smithi163 ceph-mon[25094]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-06-18T02:40:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:51 smithi163 ceph-mon[25094]: from='client.? 172.21.15.186:0/3537997450' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9bb0e529-2a21-425f-9da9-aa81ae22a52c"}]: dispatch 2024-06-18T02:40:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:51 smithi163 ceph-mon[25094]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9bb0e529-2a21-425f-9da9-aa81ae22a52c"}]: dispatch 2024-06-18T02:40:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:51 smithi163 ceph-mon[25094]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9bb0e529-2a21-425f-9da9-aa81ae22a52c"}]': finished 2024-06-18T02:40:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:51 smithi163 ceph-mon[25094]: osdmap e36: 6 total, 5 up, 6 in 2024-06-18T02:40:51.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:51 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:40:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:51 smithi186 ceph-mon[25944]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-06-18T02:40:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:51 smithi186 ceph-mon[25944]: from='client.? 172.21.15.186:0/3537997450' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9bb0e529-2a21-425f-9da9-aa81ae22a52c"}]: dispatch 2024-06-18T02:40:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:51 smithi186 ceph-mon[25944]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9bb0e529-2a21-425f-9da9-aa81ae22a52c"}]: dispatch 2024-06-18T02:40:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:51 smithi186 ceph-mon[25944]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9bb0e529-2a21-425f-9da9-aa81ae22a52c"}]': finished 2024-06-18T02:40:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:51 smithi186 ceph-mon[25944]: osdmap e36: 6 total, 5 up, 6 in 2024-06-18T02:40:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:51 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:40:52.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:52 smithi163 ceph-mon[25094]: from='client.? 172.21.15.186:0/2745264219' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:40:52.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:52 smithi163 ceph-mon[29791]: from='client.? 172.21.15.186:0/2745264219' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:40:52.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:52 smithi186 ceph-mon[25944]: from='client.? 172.21.15.186:0/2745264219' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:40:53.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:53 smithi163 ceph-mon[29791]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-18T02:40:53.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:53 smithi163 ceph-mon[25094]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-18T02:40:53.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:53 smithi186 ceph-mon[25944]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-18T02:40:54.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:54 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:40:54.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:54 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:40:54.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:54 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:40:54.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:54 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:40:54.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:54 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:40:54.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:54 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:40:55.565 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:55 smithi186 ceph-mon[25944]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-06-18T02:40:55.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:55 smithi163 ceph-mon[29791]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-06-18T02:40:55.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:55 smithi163 ceph-mon[25094]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-06-18T02:40:57.747 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:57 smithi186 ceph-mon[25944]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:40:57.747 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:57 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-18T02:40:57.747 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:57 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:57.747 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:57 smithi186 ceph-mon[25944]: Deploying daemon osd.5 on smithi186 2024-06-18T02:40:57.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:57 smithi163 ceph-mon[25094]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:40:57.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-18T02:40:57.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:57.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:57 smithi163 ceph-mon[25094]: Deploying daemon osd.5 on smithi186 2024-06-18T02:40:57.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:57 smithi163 ceph-mon[29791]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:40:57.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:57 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-18T02:40:57.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:57 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:40:57.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:57 smithi163 ceph-mon[29791]: Deploying daemon osd.5 on smithi186 2024-06-18T02:40:59.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:40:59 smithi163 ceph-mon[25094]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:40:59.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:40:59 smithi163 ceph-mon[29791]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:40:59.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:40:59 smithi186 ceph-mon[25944]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:41:01.431 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:01 smithi186 ceph-mon[25944]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:41:01.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:01 smithi163 ceph-mon[25094]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:41:01.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:01 smithi163 ceph-mon[29791]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:41:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:03 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:03 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:03 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:03 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:03 smithi186 ceph-mon[25944]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:41:03.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:03 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:03.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:03 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:03.589 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:03 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:03.589 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:03 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:03.589 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:03 smithi163 ceph-mon[25094]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:41:03.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:03 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:03.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:03 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:03.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:03 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:03.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:03 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:03.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:03 smithi163 ceph-mon[29791]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:41:05.616 INFO:teuthology.orchestra.run.smithi186.stdout:Created osd(s) 5 on host 'smithi186' 2024-06-18T02:41:05.906 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:05 smithi186 ceph-mon[25944]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:41:05.906 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:05 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:05.906 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:05 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:05.906 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:05 smithi186 ceph-mon[25944]: from='osd.5 [v2:172.21.15.186:6808/1758819799,v1:172.21.15.186:6809/1758819799]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-18T02:41:05.906 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:05 smithi186 ceph-mon[25944]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-18T02:41:05.906 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:05 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:05.906 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:05 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:05.906 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:05 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:05.906 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:05 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:06.212 DEBUG:teuthology.orchestra.run.smithi186:osd.5> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.5.service 2024-06-18T02:41:06.215 INFO:tasks.cephadm:Deploying osd.6 on smithi186 with /dev/vg_nvme/lv_2... 2024-06-18T02:41:06.215 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-06-18T02:41:06.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[25094]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:41:06.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:06.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:06.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[25094]: from='osd.5 [v2:172.21.15.186:6808/1758819799,v1:172.21.15.186:6809/1758819799]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-18T02:41:06.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[25094]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-18T02:41:06.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:06.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:06.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:06.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:06.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[29791]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:41:06.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:06.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:06.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[29791]: from='osd.5 [v2:172.21.15.186:6808/1758819799,v1:172.21.15.186:6809/1758819799]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-18T02:41:06.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[29791]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-18T02:41:06.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:06.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:06.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:06.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:05 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:07 smithi186 ceph-mon[25944]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-18T02:41:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:07 smithi186 ceph-mon[25944]: osdmap e37: 6 total, 5 up, 6 in 2024-06-18T02:41:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:07 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:41:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:07 smithi186 ceph-mon[25944]: from='osd.5 [v2:172.21.15.186:6808/1758819799,v1:172.21.15.186:6809/1758819799]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:07 smithi186 ceph-mon[25944]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:07 smithi186 ceph-mon[25944]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:41:07.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 02:41:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[32210]: 2024-06-18T02:41:07.269+0000 7f786f1a3700 -1 osd.5 0 waiting for initial osdmap 2024-06-18T02:41:07.319 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 02:41:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[32210]: 2024-06-18T02:41:07.273+0000 7f786b33d700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T02:41:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:07 smithi163 ceph-mon[25094]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-18T02:41:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:07 smithi163 ceph-mon[25094]: osdmap e37: 6 total, 5 up, 6 in 2024-06-18T02:41:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:07 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:41:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:07 smithi163 ceph-mon[25094]: from='osd.5 [v2:172.21.15.186:6808/1758819799,v1:172.21.15.186:6809/1758819799]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:07 smithi163 ceph-mon[25094]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:07 smithi163 ceph-mon[25094]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:41:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:07 smithi163 ceph-mon[29791]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-18T02:41:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:07 smithi163 ceph-mon[29791]: osdmap e37: 6 total, 5 up, 6 in 2024-06-18T02:41:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:07 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:41:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:07 smithi163 ceph-mon[29791]: from='osd.5 [v2:172.21.15.186:6808/1758819799,v1:172.21.15.186:6809/1758819799]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:07 smithi163 ceph-mon[29791]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:07 smithi163 ceph-mon[29791]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-18T02:41:08.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:08 smithi163 ceph-mon[25094]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-06-18T02:41:08.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:08 smithi163 ceph-mon[25094]: osdmap e38: 6 total, 5 up, 6 in 2024-06-18T02:41:08.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:08 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:41:08.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:08 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:41:08.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:08 smithi163 ceph-mon[29791]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-06-18T02:41:08.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:08 smithi163 ceph-mon[29791]: osdmap e38: 6 total, 5 up, 6 in 2024-06-18T02:41:08.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:08 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:41:08.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:08 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:41:08.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:08 smithi186 ceph-mon[25944]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-06-18T02:41:08.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:08 smithi186 ceph-mon[25944]: osdmap e38: 6 total, 5 up, 6 in 2024-06-18T02:41:08.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:08 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:41:08.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:08 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:41:09.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:09 smithi163 ceph-mon[25094]: purged_snaps scrub starts 2024-06-18T02:41:09.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:09 smithi163 ceph-mon[25094]: purged_snaps scrub ok 2024-06-18T02:41:09.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:09 smithi163 ceph-mon[25094]: osd.5 [v2:172.21.15.186:6808/1758819799,v1:172.21.15.186:6809/1758819799] boot 2024-06-18T02:41:09.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:09 smithi163 ceph-mon[25094]: osdmap e39: 6 total, 6 up, 6 in 2024-06-18T02:41:09.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:09 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:41:09.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:09 smithi163 ceph-mon[25094]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:09.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:09 smithi163 ceph-mon[29791]: purged_snaps scrub starts 2024-06-18T02:41:09.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:09 smithi163 ceph-mon[29791]: purged_snaps scrub ok 2024-06-18T02:41:09.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:09 smithi163 ceph-mon[29791]: osd.5 [v2:172.21.15.186:6808/1758819799,v1:172.21.15.186:6809/1758819799] boot 2024-06-18T02:41:09.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:09 smithi163 ceph-mon[29791]: osdmap e39: 6 total, 6 up, 6 in 2024-06-18T02:41:09.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:09 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:41:09.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:09 smithi163 ceph-mon[29791]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:09.561 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:09 smithi186 ceph-mon[25944]: purged_snaps scrub starts 2024-06-18T02:41:09.561 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:09 smithi186 ceph-mon[25944]: purged_snaps scrub ok 2024-06-18T02:41:09.561 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:09 smithi186 ceph-mon[25944]: osd.5 [v2:172.21.15.186:6808/1758819799,v1:172.21.15.186:6809/1758819799] boot 2024-06-18T02:41:09.562 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:09 smithi186 ceph-mon[25944]: osdmap e39: 6 total, 6 up, 6 in 2024-06-18T02:41:09.562 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:09 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:41:09.562 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:09 smithi186 ceph-mon[25944]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:09.928 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-06-18T02:41:09.929 INFO:teuthology.orchestra.run.smithi186.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-06-18T02:41:09.929 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10+0 records in 2024-06-18T02:41:09.929 INFO:teuthology.orchestra.run.smithi186.stderr:10+0 records out 2024-06-18T02:41:09.929 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0126768 s, 827 MB/s 2024-06-18T02:41:09.929 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping successful for: 2024-06-18T02:41:10.493 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:10 smithi186 ceph-mon[25944]: osdmap e40: 6 total, 6 up, 6 in 2024-06-18T02:41:10.493 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:10 smithi186 ceph-mon[25944]: Detected new or changed devices on smithi186 2024-06-18T02:41:10.493 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:10 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:10.493 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:10 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:10.493 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:10 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:10.494 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:10 smithi186 ceph-mon[25944]: Adjusting osd_memory_target on smithi186 to 8544M 2024-06-18T02:41:10.494 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:10 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:10.494 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:10 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:10.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[25094]: osdmap e40: 6 total, 6 up, 6 in 2024-06-18T02:41:10.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[25094]: Detected new or changed devices on smithi186 2024-06-18T02:41:10.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:10.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:10.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:10.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[25094]: Adjusting osd_memory_target on smithi186 to 8544M 2024-06-18T02:41:10.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:10.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[29791]: osdmap e40: 6 total, 6 up, 6 in 2024-06-18T02:41:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[29791]: Detected new or changed devices on smithi186 2024-06-18T02:41:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[29791]: Adjusting osd_memory_target on smithi186 to 8544M 2024-06-18T02:41:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:10 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:10.759 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi186:vg_nvme/lv_2 2024-06-18T02:41:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:11 smithi186 ceph-mon[25944]: osdmap e41: 6 total, 6 up, 6 in 2024-06-18T02:41:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:11 smithi186 ceph-mon[25944]: pgmap v129: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:11.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:11 smithi163 ceph-mon[29791]: osdmap e41: 6 total, 6 up, 6 in 2024-06-18T02:41:11.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:11 smithi163 ceph-mon[29791]: pgmap v129: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:11.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:11 smithi163 ceph-mon[25094]: osdmap e41: 6 total, 6 up, 6 in 2024-06-18T02:41:11.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:11 smithi163 ceph-mon[25094]: pgmap v129: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:12.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:12 smithi186 ceph-mon[25944]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-18T02:41:12.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:12 smithi186 ceph-mon[25944]: osdmap e42: 6 total, 6 up, 6 in 2024-06-18T02:41:12.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:12 smithi163 ceph-mon[29791]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-18T02:41:12.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:12 smithi163 ceph-mon[29791]: osdmap e42: 6 total, 6 up, 6 in 2024-06-18T02:41:12.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:12 smithi163 ceph-mon[25094]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-18T02:41:12.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:12 smithi163 ceph-mon[25094]: osdmap e42: 6 total, 6 up, 6 in 2024-06-18T02:41:13.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:13 smithi186 ceph-mon[25944]: osdmap e43: 6 total, 6 up, 6 in 2024-06-18T02:41:13.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:13 smithi186 ceph-mon[25944]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:13.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:13 smithi186 ceph-mon[25944]: from='client.24251 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:41:13.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:13 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:41:13.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:13 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:41:13.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:13 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:13.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:13 smithi163 ceph-mon[29791]: osdmap e43: 6 total, 6 up, 6 in 2024-06-18T02:41:13.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:13 smithi163 ceph-mon[29791]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:13.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:13 smithi163 ceph-mon[29791]: from='client.24251 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:41:13.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:13 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:41:13.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:13 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:41:13.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:13 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:13.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:13 smithi163 ceph-mon[25094]: osdmap e43: 6 total, 6 up, 6 in 2024-06-18T02:41:13.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:13 smithi163 ceph-mon[25094]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:13.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:13 smithi163 ceph-mon[25094]: from='client.24251 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:41:13.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:13 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:41:13.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:13 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:41:13.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:13 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:14 smithi163 ceph-mon[29791]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-18T02:41:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:14 smithi163 ceph-mon[29791]: Cluster is now healthy 2024-06-18T02:41:14.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:14 smithi163 ceph-mon[25094]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-18T02:41:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:14 smithi163 ceph-mon[25094]: Cluster is now healthy 2024-06-18T02:41:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:14 smithi186 ceph-mon[25944]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-18T02:41:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:14 smithi186 ceph-mon[25944]: Cluster is now healthy 2024-06-18T02:41:15.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[29791]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:15.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[29791]: from='client.? 172.21.15.186:0/1153110344' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "002ed2bc-653a-475e-9905-cd5e23dd10c3"}]: dispatch 2024-06-18T02:41:15.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[29791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "002ed2bc-653a-475e-9905-cd5e23dd10c3"}]: dispatch 2024-06-18T02:41:15.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[29791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "002ed2bc-653a-475e-9905-cd5e23dd10c3"}]': finished 2024-06-18T02:41:15.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[29791]: osdmap e44: 7 total, 6 up, 7 in 2024-06-18T02:41:15.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[29791]: from='client.? 172.21.15.186:0/3868158847' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:41:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[25094]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[25094]: from='client.? 172.21.15.186:0/1153110344' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "002ed2bc-653a-475e-9905-cd5e23dd10c3"}]: dispatch 2024-06-18T02:41:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[25094]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "002ed2bc-653a-475e-9905-cd5e23dd10c3"}]: dispatch 2024-06-18T02:41:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[25094]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "002ed2bc-653a-475e-9905-cd5e23dd10c3"}]': finished 2024-06-18T02:41:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[25094]: osdmap e44: 7 total, 6 up, 7 in 2024-06-18T02:41:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:15 smithi163 ceph-mon[25094]: from='client.? 172.21.15.186:0/3868158847' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:41:15.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:15 smithi186 ceph-mon[25944]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:15 smithi186 ceph-mon[25944]: from='client.? 172.21.15.186:0/1153110344' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "002ed2bc-653a-475e-9905-cd5e23dd10c3"}]: dispatch 2024-06-18T02:41:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:15 smithi186 ceph-mon[25944]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "002ed2bc-653a-475e-9905-cd5e23dd10c3"}]: dispatch 2024-06-18T02:41:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:15 smithi186 ceph-mon[25944]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "002ed2bc-653a-475e-9905-cd5e23dd10c3"}]': finished 2024-06-18T02:41:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:15 smithi186 ceph-mon[25944]: osdmap e44: 7 total, 6 up, 7 in 2024-06-18T02:41:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:15 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:15 smithi186 ceph-mon[25944]: from='client.? 172.21.15.186:0/3868158847' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:41:17.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:17 smithi163 ceph-mon[25094]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-18T02:41:17.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:17 smithi163 ceph-mon[29791]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-18T02:41:17.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:17 smithi186 ceph-mon[25944]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-18T02:41:19.605 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:19 smithi186 ceph-mon[25944]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-06-18T02:41:19.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:19 smithi163 ceph-mon[25094]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-06-18T02:41:19.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:19 smithi163 ceph-mon[29791]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-06-18T02:41:21.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:21 smithi163 ceph-mon[29791]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-06-18T02:41:21.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:21 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-18T02:41:21.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:21 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:21.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:21 smithi163 ceph-mon[29791]: Deploying daemon osd.6 on smithi186 2024-06-18T02:41:21.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:21 smithi163 ceph-mon[25094]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-06-18T02:41:21.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:21 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-18T02:41:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:21 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:21 smithi163 ceph-mon[25094]: Deploying daemon osd.6 on smithi186 2024-06-18T02:41:21.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:21 smithi186 ceph-mon[25944]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-06-18T02:41:21.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:21 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-18T02:41:21.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:21 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:21.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:21 smithi186 ceph-mon[25944]: Deploying daemon osd.6 on smithi186 2024-06-18T02:41:23.698 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:23 smithi186 ceph-mon[25944]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-18T02:41:23.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:23 smithi163 ceph-mon[25094]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-18T02:41:23.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:23 smithi163 ceph-mon[29791]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-18T02:41:25.541 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:25 smithi186 ceph-mon[25944]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:25.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:25 smithi163 ceph-mon[25094]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:25.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:25 smithi163 ceph-mon[29791]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:26.689 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:26 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:26.689 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:26 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:26.689 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:26 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:26.689 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:26 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:26 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:26 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:26 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:26 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:26.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:26 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:26.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:26 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:26 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:26 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:28.306 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:28 smithi186 ceph-mon[25944]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:28.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:28 smithi163 ceph-mon[25094]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:28.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:28 smithi163 ceph-mon[29791]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:29.477 INFO:teuthology.orchestra.run.smithi186.stdout:Created osd(s) 6 on host 'smithi186' 2024-06-18T02:41:29.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:29 smithi186 ceph-mon[25944]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:29 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:29 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:29 smithi186 ceph-mon[25944]: from='osd.6 [v2:172.21.15.186:6816/3657102327,v1:172.21.15.186:6817/3657102327]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-18T02:41:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:29 smithi186 ceph-mon[25944]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-18T02:41:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:29 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:29 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:29 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:29 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:30.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[29791]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:30.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:30.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:30.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[29791]: from='osd.6 [v2:172.21.15.186:6816/3657102327,v1:172.21.15.186:6817/3657102327]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-18T02:41:30.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[29791]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-18T02:41:30.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:30.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:30.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:30.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:30.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[25094]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:30.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:30.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:30.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[25094]: from='osd.6 [v2:172.21.15.186:6816/3657102327,v1:172.21.15.186:6817/3657102327]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-18T02:41:30.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[25094]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-18T02:41:30.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:30.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:30.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:30.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:29 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:30.137 DEBUG:teuthology.orchestra.run.smithi186:osd.6> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.6.service 2024-06-18T02:41:30.139 INFO:tasks.cephadm:Deploying osd.7 on smithi186 with /dev/vg_nvme/lv_1... 2024-06-18T02:41:30.140 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-06-18T02:41:31.327 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:31 smithi186 ceph-mon[25944]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-18T02:41:31.328 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:31 smithi186 ceph-mon[25944]: osdmap e45: 7 total, 6 up, 7 in 2024-06-18T02:41:31.328 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:31 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:31.328 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:31 smithi186 ceph-mon[25944]: from='osd.6 [v2:172.21.15.186:6816/3657102327,v1:172.21.15.186:6817/3657102327]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:31.328 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:31 smithi186 ceph-mon[25944]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:31.328 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 02:41:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[35079]: 2024-06-18T02:41:31.070+0000 7f0df8cfa700 -1 osd.6 0 waiting for initial osdmap 2024-06-18T02:41:31.328 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 02:41:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[35079]: 2024-06-18T02:41:31.077+0000 7f0df1e8e700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T02:41:31.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:31 smithi163 ceph-mon[25094]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-18T02:41:31.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:31 smithi163 ceph-mon[25094]: osdmap e45: 7 total, 6 up, 7 in 2024-06-18T02:41:31.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:31 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:31.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:31 smithi163 ceph-mon[25094]: from='osd.6 [v2:172.21.15.186:6816/3657102327,v1:172.21.15.186:6817/3657102327]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:31.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:31 smithi163 ceph-mon[25094]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:31.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:31 smithi163 ceph-mon[29791]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-18T02:41:31.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:31 smithi163 ceph-mon[29791]: osdmap e45: 7 total, 6 up, 7 in 2024-06-18T02:41:31.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:31 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:31.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:31 smithi163 ceph-mon[29791]: from='osd.6 [v2:172.21.15.186:6816/3657102327,v1:172.21.15.186:6817/3657102327]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:31.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:31 smithi163 ceph-mon[29791]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:32.398 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:32 smithi186 ceph-mon[25944]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:32.398 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:32 smithi186 ceph-mon[25944]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-06-18T02:41:32.398 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:32 smithi186 ceph-mon[25944]: osdmap e46: 7 total, 6 up, 7 in 2024-06-18T02:41:32.398 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:32 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:32.398 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:32 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:32.398 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:32 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:32.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:32 smithi163 ceph-mon[25094]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:32.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:32 smithi163 ceph-mon[25094]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-06-18T02:41:32.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:32 smithi163 ceph-mon[25094]: osdmap e46: 7 total, 6 up, 7 in 2024-06-18T02:41:32.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:32 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:32.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:32 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:32 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:32.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:32 smithi163 ceph-mon[29791]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:32.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:32 smithi163 ceph-mon[29791]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-06-18T02:41:32.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:32 smithi163 ceph-mon[29791]: osdmap e46: 7 total, 6 up, 7 in 2024-06-18T02:41:32.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:32 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:32.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:32 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:32.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:32 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:33.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:33 smithi186 ceph-mon[25944]: purged_snaps scrub starts 2024-06-18T02:41:33.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:33 smithi186 ceph-mon[25944]: purged_snaps scrub ok 2024-06-18T02:41:33.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:33 smithi186 ceph-mon[25944]: osd.6 [v2:172.21.15.186:6816/3657102327,v1:172.21.15.186:6817/3657102327] boot 2024-06-18T02:41:33.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:33 smithi186 ceph-mon[25944]: osdmap e47: 7 total, 7 up, 7 in 2024-06-18T02:41:33.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:33 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:33.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:33 smithi163 ceph-mon[29791]: purged_snaps scrub starts 2024-06-18T02:41:33.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:33 smithi163 ceph-mon[29791]: purged_snaps scrub ok 2024-06-18T02:41:33.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:33 smithi163 ceph-mon[29791]: osd.6 [v2:172.21.15.186:6816/3657102327,v1:172.21.15.186:6817/3657102327] boot 2024-06-18T02:41:33.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:33 smithi163 ceph-mon[29791]: osdmap e47: 7 total, 7 up, 7 in 2024-06-18T02:41:33.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:33 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:33.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:33 smithi163 ceph-mon[25094]: purged_snaps scrub starts 2024-06-18T02:41:33.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:33 smithi163 ceph-mon[25094]: purged_snaps scrub ok 2024-06-18T02:41:33.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:33 smithi163 ceph-mon[25094]: osd.6 [v2:172.21.15.186:6816/3657102327,v1:172.21.15.186:6817/3657102327] boot 2024-06-18T02:41:33.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:33 smithi163 ceph-mon[25094]: osdmap e47: 7 total, 7 up, 7 in 2024-06-18T02:41:33.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:33 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:41:34.102 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-06-18T02:41:34.102 INFO:teuthology.orchestra.run.smithi186.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-06-18T02:41:34.102 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10+0 records in 2024-06-18T02:41:34.103 INFO:teuthology.orchestra.run.smithi186.stderr:10+0 records out 2024-06-18T02:41:34.103 INFO:teuthology.orchestra.run.smithi186.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0116369 s, 901 MB/s 2024-06-18T02:41:34.103 INFO:teuthology.orchestra.run.smithi186.stderr:--> Zapping successful for: 2024-06-18T02:41:34.484 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:34 smithi186 ceph-mon[25944]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:34.484 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:34 smithi186 ceph-mon[25944]: osdmap e48: 7 total, 7 up, 7 in 2024-06-18T02:41:34.484 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:34 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:34.484 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:34 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:34.484 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:34 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:34.484 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:34 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:34.484 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:34 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:34.484 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:34 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:34.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[29791]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:34.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[29791]: osdmap e48: 7 total, 7 up, 7 in 2024-06-18T02:41:34.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:34.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:34.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:34.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:34.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:34.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:34.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[25094]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-18T02:41:34.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[25094]: osdmap e48: 7 total, 7 up, 7 in 2024-06-18T02:41:34.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:34.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:34.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:34.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:34.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:34.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:34 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:34.774 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi186:vg_nvme/lv_1 2024-06-18T02:41:35.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:35 smithi186 ceph-mon[25944]: Detected new or changed devices on smithi186 2024-06-18T02:41:35.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:35 smithi186 ceph-mon[25944]: Adjusting osd_memory_target on smithi186 to 5696M 2024-06-18T02:41:35.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:35 smithi186 ceph-mon[25944]: osdmap e49: 7 total, 7 up, 7 in 2024-06-18T02:41:35.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:35 smithi163 ceph-mon[25094]: Detected new or changed devices on smithi186 2024-06-18T02:41:35.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:35 smithi163 ceph-mon[25094]: Adjusting osd_memory_target on smithi186 to 5696M 2024-06-18T02:41:35.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:35 smithi163 ceph-mon[25094]: osdmap e49: 7 total, 7 up, 7 in 2024-06-18T02:41:35.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:35 smithi163 ceph-mon[29791]: Detected new or changed devices on smithi186 2024-06-18T02:41:35.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:35 smithi163 ceph-mon[29791]: Adjusting osd_memory_target on smithi186 to 5696M 2024-06-18T02:41:35.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:35 smithi163 ceph-mon[29791]: osdmap e49: 7 total, 7 up, 7 in 2024-06-18T02:41:36.340 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:36 smithi186 ceph-mon[25944]: pgmap v149: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-06-18T02:41:36.340 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:36 smithi186 ceph-mon[25944]: osdmap e50: 7 total, 7 up, 7 in 2024-06-18T02:41:36.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:36 smithi163 ceph-mon[25094]: pgmap v149: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-06-18T02:41:36.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:36 smithi163 ceph-mon[25094]: osdmap e50: 7 total, 7 up, 7 in 2024-06-18T02:41:36.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:36 smithi163 ceph-mon[29791]: pgmap v149: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-06-18T02:41:36.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:36 smithi163 ceph-mon[29791]: osdmap e50: 7 total, 7 up, 7 in 2024-06-18T02:41:37.353 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:37 smithi186 ceph-mon[25944]: osdmap e51: 7 total, 7 up, 7 in 2024-06-18T02:41:37.353 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:37 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:41:37.353 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:37 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:41:37.354 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:37 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:37.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:37 smithi163 ceph-mon[25094]: osdmap e51: 7 total, 7 up, 7 in 2024-06-18T02:41:37.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:37 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:41:37.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:37 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:41:37.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:37 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:37.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:37 smithi163 ceph-mon[29791]: osdmap e51: 7 total, 7 up, 7 in 2024-06-18T02:41:37.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:37 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-18T02:41:37.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:37 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-18T02:41:37.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:37 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:38.404 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:38 smithi186 ceph-mon[25944]: pgmap v152: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-06-18T02:41:38.404 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:38 smithi186 ceph-mon[25944]: from='client.24278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:41:38.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:38 smithi163 ceph-mon[25094]: pgmap v152: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-06-18T02:41:38.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:38 smithi163 ceph-mon[25094]: from='client.24278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:41:38.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:38 smithi163 ceph-mon[29791]: pgmap v152: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-06-18T02:41:38.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:38 smithi163 ceph-mon[29791]: from='client.24278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi186:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:41:39.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:39 smithi163 ceph-mon[29791]: from='client.? 172.21.15.186:0/3445818526' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d79629d8-851c-4e0d-b1da-752a6505037a"}]: dispatch 2024-06-18T02:41:39.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:39 smithi163 ceph-mon[29791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d79629d8-851c-4e0d-b1da-752a6505037a"}]: dispatch 2024-06-18T02:41:39.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:39 smithi163 ceph-mon[29791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d79629d8-851c-4e0d-b1da-752a6505037a"}]': finished 2024-06-18T02:41:39.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:39 smithi163 ceph-mon[29791]: osdmap e52: 8 total, 7 up, 8 in 2024-06-18T02:41:39.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:39 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:39.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:39 smithi163 ceph-mon[25094]: from='client.? 172.21.15.186:0/3445818526' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d79629d8-851c-4e0d-b1da-752a6505037a"}]: dispatch 2024-06-18T02:41:39.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:39 smithi163 ceph-mon[25094]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d79629d8-851c-4e0d-b1da-752a6505037a"}]: dispatch 2024-06-18T02:41:39.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:39 smithi163 ceph-mon[25094]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d79629d8-851c-4e0d-b1da-752a6505037a"}]': finished 2024-06-18T02:41:39.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:39 smithi163 ceph-mon[25094]: osdmap e52: 8 total, 7 up, 8 in 2024-06-18T02:41:39.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:39 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:39.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:39 smithi186 ceph-mon[25944]: from='client.? 172.21.15.186:0/3445818526' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d79629d8-851c-4e0d-b1da-752a6505037a"}]: dispatch 2024-06-18T02:41:39.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:39 smithi186 ceph-mon[25944]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d79629d8-851c-4e0d-b1da-752a6505037a"}]: dispatch 2024-06-18T02:41:39.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:39 smithi186 ceph-mon[25944]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d79629d8-851c-4e0d-b1da-752a6505037a"}]': finished 2024-06-18T02:41:39.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:39 smithi186 ceph-mon[25944]: osdmap e52: 8 total, 7 up, 8 in 2024-06-18T02:41:39.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:39 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:40.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:40 smithi163 ceph-mon[29791]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-06-18T02:41:40.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:40 smithi163 ceph-mon[29791]: from='client.? 172.21.15.186:0/2799118722' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:41:40.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:40 smithi163 ceph-mon[25094]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-06-18T02:41:40.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:40 smithi163 ceph-mon[25094]: from='client.? 172.21.15.186:0/2799118722' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:41:40.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:40 smithi186 ceph-mon[25944]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-06-18T02:41:40.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:40 smithi186 ceph-mon[25944]: from='client.? 172.21.15.186:0/2799118722' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-18T02:41:41.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:41 smithi163 ceph-mon[29791]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:41.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:41 smithi163 ceph-mon[25094]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:41.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:41 smithi186 ceph-mon[25944]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:43.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:43 smithi163 ceph-mon[25094]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:43.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:43 smithi163 ceph-mon[29791]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:43.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:43 smithi186 ceph-mon[25944]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:45.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:45 smithi186 ceph-mon[25944]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:45.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:45 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-18T02:41:45.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:45 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:45.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:45 smithi186 ceph-mon[25944]: Deploying daemon osd.7 on smithi186 2024-06-18T02:41:45.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:45 smithi163 ceph-mon[25094]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:45.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:45 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-18T02:41:45.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:45 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:45.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:45 smithi163 ceph-mon[25094]: Deploying daemon osd.7 on smithi186 2024-06-18T02:41:45.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:45 smithi163 ceph-mon[29791]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:45.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:45 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-18T02:41:45.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:45 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:45.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:45 smithi163 ceph-mon[29791]: Deploying daemon osd.7 on smithi186 2024-06-18T02:41:47.763 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:47 smithi186 ceph-mon[25944]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:47.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:47 smithi163 ceph-mon[25094]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:47.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:47 smithi163 ceph-mon[29791]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:49.762 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:49 smithi186 ceph-mon[25944]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:49.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:49 smithi163 ceph-mon[25094]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:49.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:49 smithi163 ceph-mon[29791]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:51.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:51 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:51.789 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:51 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:51.789 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:51 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:51.789 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:51 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:51.789 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:51 smithi186 ceph-mon[25944]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:51.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:51 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:51.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:51 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:51.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:51 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:51.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:51 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:51.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:51 smithi163 ceph-mon[25094]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:51.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:51 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:51.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:51 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:51.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:51 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:51.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:51 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:51.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:51 smithi163 ceph-mon[29791]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:53.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:53 smithi186 ceph-mon[25944]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:53.618 INFO:teuthology.orchestra.run.smithi186.stdout:Created osd(s) 7 on host 'smithi186' 2024-06-18T02:41:53.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:53 smithi163 ceph-mon[25094]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:53.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:53 smithi163 ceph-mon[29791]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:54.203 DEBUG:teuthology.orchestra.run.smithi186:osd.7> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.7.service 2024-06-18T02:41:54.205 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-06-18T02:41:54.205 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd stat -f json 2024-06-18T02:41:54.524 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:54 smithi163 ceph-mon[29791]: from='osd.7 [v2:172.21.15.186:6824/894630858,v1:172.21.15.186:6825/894630858]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-18T02:41:54.524 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:54 smithi163 ceph-mon[29791]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-18T02:41:54.524 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:54 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:54.524 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:54 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:54.524 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:54 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:41:54.524 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:54 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:41:54.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:54 smithi163 ceph-mon[25094]: from='osd.7 [v2:172.21.15.186:6824/894630858,v1:172.21.15.186:6825/894630858]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-18T02:41:54.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:54 smithi163 ceph-mon[25094]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-18T02:41:54.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:54 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:54.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:54 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:54.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:54 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:41:54.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:54 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:41:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:54 smithi186 ceph-mon[25944]: from='osd.7 [v2:172.21.15.186:6824/894630858,v1:172.21.15.186:6825/894630858]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-18T02:41:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:54 smithi186 ceph-mon[25944]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-18T02:41:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:54 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:54 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:54 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:41:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:54 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:41:55.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:55 smithi186 ceph-mon[25944]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:55.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:55 smithi186 ceph-mon[25944]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-18T02:41:55.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:55 smithi186 ceph-mon[25944]: osdmap e53: 8 total, 7 up, 8 in 2024-06-18T02:41:55.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:55 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:55.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:55 smithi186 ceph-mon[25944]: from='osd.7 [v2:172.21.15.186:6824/894630858,v1:172.21.15.186:6825/894630858]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:55.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:55 smithi186 ceph-mon[25944]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:55.568 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 02:41:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[37987]: 2024-06-18T02:41:55.481+0000 7f49b27ab700 -1 osd.7 0 waiting for initial osdmap 2024-06-18T02:41:55.569 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 02:41:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[37987]: 2024-06-18T02:41:55.488+0000 7f49ad943700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T02:41:55.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:55 smithi163 ceph-mon[29791]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:55.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:55 smithi163 ceph-mon[29791]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-18T02:41:55.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:55 smithi163 ceph-mon[29791]: osdmap e53: 8 total, 7 up, 8 in 2024-06-18T02:41:55.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:55 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:55.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:55 smithi163 ceph-mon[29791]: from='osd.7 [v2:172.21.15.186:6824/894630858,v1:172.21.15.186:6825/894630858]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:55.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:55 smithi163 ceph-mon[29791]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:55.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:55 smithi163 ceph-mon[25094]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:55.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:55 smithi163 ceph-mon[25094]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-18T02:41:55.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:55 smithi163 ceph-mon[25094]: osdmap e53: 8 total, 7 up, 8 in 2024-06-18T02:41:55.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:55 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:55.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:55 smithi163 ceph-mon[25094]: from='osd.7 [v2:172.21.15.186:6824/894630858,v1:172.21.15.186:6825/894630858]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:55.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:55 smithi163 ceph-mon[25094]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T02:41:56.132 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:41:56.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[29791]: purged_snaps scrub starts 2024-06-18T02:41:56.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[29791]: purged_snaps scrub ok 2024-06-18T02:41:56.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[29791]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-06-18T02:41:56.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[29791]: osdmap e54: 8 total, 7 up, 8 in 2024-06-18T02:41:56.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:56.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:56.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3921653953' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-18T02:41:56.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[25094]: purged_snaps scrub starts 2024-06-18T02:41:56.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[25094]: purged_snaps scrub ok 2024-06-18T02:41:56.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[25094]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-06-18T02:41:56.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[25094]: osdmap e54: 8 total, 7 up, 8 in 2024-06-18T02:41:56.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:56.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:56.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:56 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3921653953' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-18T02:41:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:56 smithi186 ceph-mon[25944]: purged_snaps scrub starts 2024-06-18T02:41:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:56 smithi186 ceph-mon[25944]: purged_snaps scrub ok 2024-06-18T02:41:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:56 smithi186 ceph-mon[25944]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]': finished 2024-06-18T02:41:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:56 smithi186 ceph-mon[25944]: osdmap e54: 8 total, 7 up, 8 in 2024-06-18T02:41:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:56 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:56 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:56 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3921653953' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-18T02:41:56.924 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1718678492,"num_in_osds":8,"osd_in_since":1718678499,"num_remapped_pgs":0} 2024-06-18T02:41:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:57 smithi186 ceph-mon[25944]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:57 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:57 smithi186 ceph-mon[25944]: osd.7 [v2:172.21.15.186:6824/894630858,v1:172.21.15.186:6825/894630858] boot 2024-06-18T02:41:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:57 smithi186 ceph-mon[25944]: osdmap e55: 8 total, 8 up, 8 in 2024-06-18T02:41:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:57 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:57.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:57 smithi163 ceph-mon[29791]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:57.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:57 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:57.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:57 smithi163 ceph-mon[29791]: osd.7 [v2:172.21.15.186:6824/894630858,v1:172.21.15.186:6825/894630858] boot 2024-06-18T02:41:57.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:57 smithi163 ceph-mon[29791]: osdmap e55: 8 total, 8 up, 8 in 2024-06-18T02:41:57.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:57 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:57.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:57 smithi163 ceph-mon[25094]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-18T02:41:57.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:57.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:57 smithi163 ceph-mon[25094]: osd.7 [v2:172.21.15.186:6824/894630858,v1:172.21.15.186:6825/894630858] boot 2024-06-18T02:41:57.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:57 smithi163 ceph-mon[25094]: osdmap e55: 8 total, 8 up, 8 in 2024-06-18T02:41:57.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:57 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:41:57.926 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd stat -f json 2024-06-18T02:41:58.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: osdmap e56: 8 total, 8 up, 8 in 2024-06-18T02:41:58.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: Detected new or changed devices on smithi186 2024-06-18T02:41:58.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: Adjusting osd_memory_target on smithi186 to 4272M 2024-06-18T02:41:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:58.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: osdmap e56: 8 total, 8 up, 8 in 2024-06-18T02:41:58.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: Detected new or changed devices on smithi186 2024-06-18T02:41:58.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:58.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:58.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:58.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:58.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:58.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: Adjusting osd_memory_target on smithi186 to 4272M 2024-06-18T02:41:58.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:58.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:58.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:58.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:58.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:58.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:58 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:58.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: osdmap e56: 8 total, 8 up, 8 in 2024-06-18T02:41:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: Detected new or changed devices on smithi186 2024-06-18T02:41:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:41:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: Adjusting osd_memory_target on smithi186 to 4272M 2024-06-18T02:41:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:41:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:41:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:41:58.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:58 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:41:59.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:59 smithi163 ceph-mon[25094]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:41:59.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:41:59 smithi163 ceph-mon[25094]: osdmap e57: 8 total, 8 up, 8 in 2024-06-18T02:41:59.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:59 smithi163 ceph-mon[29791]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:41:59.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:41:59 smithi163 ceph-mon[29791]: osdmap e57: 8 total, 8 up, 8 in 2024-06-18T02:41:59.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:59 smithi186 ceph-mon[25944]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:41:59.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:41:59 smithi186 ceph-mon[25944]: osdmap e57: 8 total, 8 up, 8 in 2024-06-18T02:41:59.849 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:42:00.526 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1718678516,"num_in_osds":8,"osd_in_since":1718678499,"num_remapped_pgs":0} 2024-06-18T02:42:00.527 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd dump --format=json 2024-06-18T02:42:00.736 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:00.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:00 smithi163 ceph-mon[25094]: osdmap e58: 8 total, 8 up, 8 in 2024-06-18T02:42:00.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:00 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/4005740993' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-18T02:42:00.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:00 smithi163 ceph-mon[29791]: osdmap e58: 8 total, 8 up, 8 in 2024-06-18T02:42:00.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:00 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/4005740993' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-18T02:42:00.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:00 smithi186 ceph-mon[25944]: osdmap e58: 8 total, 8 up, 8 in 2024-06-18T02:42:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:00 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/4005740993' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-18T02:42:01.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:01 smithi163 ceph-mon[25094]: pgmap v171: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-06-18T02:42:01.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:01 smithi163 ceph-mon[25094]: osdmap e59: 8 total, 8 up, 8 in 2024-06-18T02:42:01.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:01 smithi163 ceph-mon[29791]: pgmap v171: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-06-18T02:42:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:01 smithi163 ceph-mon[29791]: osdmap e59: 8 total, 8 up, 8 in 2024-06-18T02:42:01.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:01 smithi186 ceph-mon[25944]: pgmap v171: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-06-18T02:42:01.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:01 smithi186 ceph-mon[25944]: osdmap e59: 8 total, 8 up, 8 in 2024-06-18T02:42:02.493 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:42:02.493 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":59,"fsid":"929affee-2d1b-11ef-bc9d-c7b262605968","created":"2024-06-18T02:37:00.382705+0000","modified":"2024-06-18T02:42:00.504454+0000","last_up_change":"2024-06-18T02:41:56.477912+0000","last_in_change":"2024-06-18T02:41:39.027963+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-06-18T02:40:00.660490+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"142df380-e27f-4351-b05c-367d07947ced","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":4251995344},{"type":"v1","addr":"172.21.15.163:6803","nonce":4251995344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":4251995344},{"type":"v1","addr":"172.21.15.163:6805","nonce":4251995344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":4251995344},{"type":"v1","addr":"172.21.15.163:6809","nonce":4251995344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":4251995344},{"type":"v1","addr":"172.21.15.163:6807","nonce":4251995344}]},"public_addr":"172.21.15.163:6803/4251995344","cluster_addr":"172.21.15.163:6805/4251995344","heartbeat_back_addr":"172.21.15.163:6809/4251995344","heartbeat_front_addr":"172.21.15.163:6807/4251995344","state":["exists","up"]},{"osd":1,"uuid":"696b47f4-f00e-4c37-80ec-6df94769a34f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":1462975684},{"type":"v1","addr":"172.21.15.163:6811","nonce":1462975684}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":1462975684},{"type":"v1","addr":"172.21.15.163:6813","nonce":1462975684}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":1462975684},{"type":"v1","addr":"172.21.15.163:6817","nonce":1462975684}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":1462975684},{"type":"v1","addr":"172.21.15.163:6815","nonce":1462975684}]},"public_addr":"172.21.15.163:6811/1462975684","cluster_addr":"172.21.15.163:6813/1462975684","heartbeat_back_addr":"172.21.15.163:6817/1462975684","heartbeat_front_addr":"172.21.15.163:6815/1462975684","state":["exists","up"]},{"osd":2,"uuid":"ad504a3c-fa5c-4b4c-a079-a66b52088d68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":923602700},{"type":"v1","addr":"172.21.15.163:6819","nonce":923602700}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":923602700},{"type":"v1","addr":"172.21.15.163:6821","nonce":923602700}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":923602700},{"type":"v1","addr":"172.21.15.163:6825","nonce":923602700}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":923602700},{"type":"v1","addr":"172.21.15.163:6823","nonce":923602700}]},"public_addr":"172.21.15.163:6819/923602700","cluster_addr":"172.21.15.163:6821/923602700","heartbeat_back_addr":"172.21.15.163:6825/923602700","heartbeat_front_addr":"172.21.15.163:6823/923602700","state":["exists","up"]},{"osd":3,"uuid":"f9f9da4f-4afc-48cf-92a2-f714ca3f4c4f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":2061575135},{"type":"v1","addr":"172.21.15.163:6827","nonce":2061575135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":2061575135},{"type":"v1","addr":"172.21.15.163:6829","nonce":2061575135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":2061575135},{"type":"v1","addr":"172.21.15.163:6833","nonce":2061575135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":2061575135},{"type":"v1","addr":"172.21.15.163:6831","nonce":2061575135}]},"public_addr":"172.21.15.163:6827/2061575135","cluster_addr":"172.21.15.163:6829/2061575135","heartbeat_back_addr":"172.21.15.163:6833/2061575135","heartbeat_front_addr":"172.21.15.163:6831/2061575135","state":["exists","up"]},{"osd":4,"uuid":"12a9d06c-9aab-4659-8533-f57336078d0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6800","nonce":368766852},{"type":"v1","addr":"172.21.15.186:6801","nonce":368766852}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6802","nonce":368766852},{"type":"v1","addr":"172.21.15.186:6803","nonce":368766852}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6806","nonce":368766852},{"type":"v1","addr":"172.21.15.186:6807","nonce":368766852}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6804","nonce":368766852},{"type":"v1","addr":"172.21.15.186:6805","nonce":368766852}]},"public_addr":"172.21.15.186:6801/368766852","cluster_addr":"172.21.15.186:6803/368766852","heartbeat_back_addr":"172.21.15.186:6807/368766852","heartbeat_front_addr":"172.21.15.186:6805/368766852","state":["exists","up"]},{"osd":5,"uuid":"9bb0e529-2a21-425f-9da9-aa81ae22a52c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6808","nonce":1758819799},{"type":"v1","addr":"172.21.15.186:6809","nonce":1758819799}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6810","nonce":1758819799},{"type":"v1","addr":"172.21.15.186:6811","nonce":1758819799}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6814","nonce":1758819799},{"type":"v1","addr":"172.21.15.186:6815","nonce":1758819799}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6812","nonce":1758819799},{"type":"v1","addr":"172.21.15.186:6813","nonce":1758819799}]},"public_addr":"172.21.15.186:6809/1758819799","cluster_addr":"172.21.15.186:6811/1758819799","heartbeat_back_addr":"172.21.15.186:6815/1758819799","heartbeat_front_addr":"172.21.15.186:6813/1758819799","state":["exists","up"]},{"osd":6,"uuid":"002ed2bc-653a-475e-9905-cd5e23dd10c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6816","nonce":3657102327},{"type":"v1","addr":"172.21.15.186:6817","nonce":3657102327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6818","nonce":3657102327},{"type":"v1","addr":"172.21.15.186:6819","nonce":3657102327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6822","nonce":3657102327},{"type":"v1","addr":"172.21.15.186:6823","nonce":3657102327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6820","nonce":3657102327},{"type":"v1","addr":"172.21.15.186:6821","nonce":3657102327}]},"public_addr":"172.21.15.186:6817/3657102327","cluster_addr":"172.21.15.186:6819/3657102327","heartbeat_back_addr":"172.21.15.186:6823/3657102327","heartbeat_front_addr":"172.21.15.186:6821/3657102327","state":["exists","up"]},{"osd":7,"uuid":"d79629d8-851c-4e0d-b1da-752a6505037a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6824","nonce":894630858},{"type":"v1","addr":"172.21.15.186:6825","nonce":894630858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6826","nonce":894630858},{"type":"v1","addr":"172.21.15.186:6827","nonce":894630858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6830","nonce":894630858},{"type":"v1","addr":"172.21.15.186:6831","nonce":894630858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6828","nonce":894630858},{"type":"v1","addr":"172.21.15.186:6829","nonce":894630858}]},"public_addr":"172.21.15.186:6825/894630858","cluster_addr":"172.21.15.186:6827/894630858","heartbeat_back_addr":"172.21.15.186:6831/894630858","heartbeat_front_addr":"172.21.15.186:6829/894630858","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:39:11.602779+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:39:34.500246+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:39:57.071084+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:40:21.371956+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:40:43.043329+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:41:06.426162+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:41:30.195208+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:41:54.520226+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/1291328983":"2024-06-19T02:37:54.311318+0000","172.21.15.163:6801/4217980061":"2024-06-19T02:37:54.311318+0000","172.21.15.163:0/3911242098":"2024-06-19T02:37:54.311318+0000","172.21.15.163:6800/396937736":"2024-06-19T02:37:32.086178+0000","172.21.15.163:6801/396937736":"2024-06-19T02:37:32.086178+0000","172.21.15.163:0/921115307":"2024-06-19T02:37:54.311318+0000","172.21.15.163:0/3175087117":"2024-06-19T02:37:32.086178+0000","172.21.15.163:0/2926912200":"2024-06-19T02:37:32.086178+0000","172.21.15.163:6800/4217980061":"2024-06-19T02:37:54.311318+0000","172.21.15.163:0/538097455":"2024-06-19T02:37:32.086178+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-06-18T02:42:02.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:02 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1745615420' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-18T02:42:02.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:02 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1745615420' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-18T02:42:02.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:02 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1745615420' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-18T02:42:03.188 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-06-18T02:40:00.660490+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-06-18T02:42:03.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-06-18T02:42:03.403 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:03.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:03 smithi163 ceph-mon[29791]: pgmap v173: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-06-18T02:42:03.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:03 smithi163 ceph-mon[25094]: pgmap v173: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-06-18T02:42:03.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:03 smithi186 ceph-mon[25944]: pgmap v173: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-06-18T02:42:05.184 INFO:teuthology.orchestra.run.smithi163.stdout:pg_num: 1 2024-06-18T02:42:05.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:05 smithi163 ceph-mon[25094]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-18T02:42:05.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:05 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1999085849' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-06-18T02:42:05.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:05 smithi163 ceph-mon[29791]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-18T02:42:05.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:05 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1999085849' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-06-18T02:42:05.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:05 smithi186 ceph-mon[25944]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-18T02:42:05.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:05 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1999085849' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-06-18T02:42:05.911 INFO:tasks.cephadm:Adding prometheus.a on smithi186 2024-06-18T02:42:05.911 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch apply prometheus '1;smithi186=a' 2024-06-18T02:42:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:07 smithi163 ceph-mon[25094]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-06-18T02:42:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:07 smithi163 ceph-mon[29791]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-06-18T02:42:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:07 smithi186 ceph-mon[25944]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-06-18T02:42:07.905 INFO:teuthology.orchestra.run.smithi186.stdout:Scheduled prometheus update... 2024-06-18T02:42:08.598 DEBUG:teuthology.orchestra.run.smithi186:prometheus.a> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a.service 2024-06-18T02:42:08.601 INFO:tasks.cephadm:Adding node-exporter.a on smithi163 2024-06-18T02:42:08.601 INFO:tasks.cephadm:Adding node-exporter.b on smithi186 2024-06-18T02:42:08.601 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch apply node-exporter '2;smithi163=a;smithi186=b' 2024-06-18T02:42:09.194 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:08 smithi186 ceph-mon[25944]: from='client.24317 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi186=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:42:09.194 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:08 smithi186 ceph-mon[25944]: Saving service prometheus spec with placement smithi186=a;count:1 2024-06-18T02:42:09.194 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:08 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:42:09.195 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:08 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:42:09.195 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:08 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:42:09.195 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:08 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:42:09.195 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:08 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:42:09.195 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:08 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-06-18T02:42:09.195 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ignoring --setuser ceph since I am not root 2024-06-18T02:42:09.195 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ignoring --setgroup ceph since I am not root 2024-06-18T02:42:09.195 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:09 smithi186 ceph-mgr[27115]: -- 172.21.15.186:0/3149585608 <== mon.2 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5558d1218340 con 0x5558d11f0c00 2024-06-18T02:42:09.195 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:09.121+0000 7f1fe99ad000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T02:42:09.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[25094]: from='client.24317 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi186=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:42:09.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[25094]: Saving service prometheus spec with placement smithi186=a;count:1 2024-06-18T02:42:09.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:42:09.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:42:09.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:42:09.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:42:09.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:42:09.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-06-18T02:42:09.253 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:09 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ignoring --setuser ceph since I am not root 2024-06-18T02:42:09.253 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:09 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ignoring --setgroup ceph since I am not root 2024-06-18T02:42:09.254 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:09 smithi163 ceph-mgr[25326]: -- 172.21.15.163:0/2384597502 <== mon.1 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ae3de7a340 con 0x55ae3de52c00 2024-06-18T02:42:09.254 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:09 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:09.127+0000 7f2f8e885000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T02:42:09.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[29791]: from='client.24317 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi186=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:42:09.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[29791]: Saving service prometheus spec with placement smithi186=a;count:1 2024-06-18T02:42:09.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:42:09.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:42:09.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:42:09.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:42:09.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' 2024-06-18T02:42:09.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:08 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-06-18T02:42:09.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:09 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:09.252+0000 7f2f8e885000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:42:09.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:09 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:09.350+0000 7f2f8e885000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T02:42:09.567 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:09.253+0000 7f1fe99ad000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:42:09.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:09.353+0000 7f1fe99ad000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T02:42:09.958 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:09 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:09.652+0000 7f2f8e885000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T02:42:09.959 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:09 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:09.733+0000 7f2f8e885000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T02:42:09.959 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:09.662+0000 7f1fe99ad000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T02:42:09.959 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:09.744+0000 7f1fe99ad000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T02:42:10.210 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:09 smithi186 ceph-mon[25944]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-06-18T02:42:10.210 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:09 smithi186 ceph-mon[25944]: mgrmap e16: y(active, since 4m), standbys: x 2024-06-18T02:42:10.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:09 smithi163 ceph-mon[29791]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-06-18T02:42:10.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:09 smithi163 ceph-mon[29791]: mgrmap e16: y(active, since 4m), standbys: x 2024-06-18T02:42:10.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:09 smithi163 ceph-mon[25094]: from='mgr.14150 172.21.15.163:0/1562694155' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-06-18T02:42:10.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:09 smithi163 ceph-mon[25094]: mgrmap e16: y(active, since 4m), standbys: x 2024-06-18T02:42:11.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:10 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:10.842+0000 7f2f8e885000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T02:42:11.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:11.006+0000 7f2f8e885000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T02:42:11.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:11.087+0000 7f2f8e885000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T02:42:11.317 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:10 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:10.885+0000 7f1fe99ad000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T02:42:11.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:11.048+0000 7f1fe99ad000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T02:42:11.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:11.128+0000 7f1fe99ad000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T02:42:11.770 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:11.440+0000 7f1fe99ad000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T02:42:11.771 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:11.529+0000 7f1fe99ad000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T02:42:11.771 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:11.620+0000 7f1fe99ad000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T02:42:11.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:11.400+0000 7f2f8e885000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T02:42:11.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:11.489+0000 7f2f8e885000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T02:42:11.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:11.582+0000 7f2f8e885000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T02:42:12.067 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:11.892+0000 7f1fe99ad000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:42:12.254 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:11.822+0000 7f2f8e885000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:42:12.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:12 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:12.254+0000 7f2f8e885000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T02:42:12.811 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:12.328+0000 7f1fe99ad000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T02:42:13.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:12 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:12.737+0000 7f2f8e885000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T02:42:13.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:12 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:12.831+0000 7f2f8e885000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T02:42:13.067 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:12.811+0000 7f1fe99ad000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T02:42:13.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:12.905+0000 7f1fe99ad000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T02:42:13.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:13.120+0000 7f2f8e885000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T02:42:13.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:13.207+0000 7f2f8e885000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T02:42:13.567 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:13.197+0000 7f1fe99ad000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T02:42:13.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:13.285+0000 7f1fe99ad000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T02:42:13.951 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:13.611+0000 7f2f8e885000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T02:42:13.951 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:13.743+0000 7f2f8e885000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T02:42:13.951 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:13.836+0000 7f2f8e885000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T02:42:13.951 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:13.951+0000 7f2f8e885000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T02:42:14.023 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:13.689+0000 7f1fe99ad000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T02:42:14.024 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:13.819+0000 7f1fe99ad000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T02:42:14.024 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:13.911+0000 7f1fe99ad000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T02:42:14.024 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:14.023+0000 7f1fe99ad000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T02:42:14.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:14 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:14.039+0000 7f2f8e885000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T02:42:14.317 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:14.113+0000 7f1fe99ad000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T02:42:14.941 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:14 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:14.583+0000 7f2f8e885000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T02:42:15.004 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:14.654+0000 7f1fe99ad000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T02:42:15.202 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-mon[29791]: Active manager daemon y restarted 2024-06-18T02:42:15.202 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-mon[29791]: Activating manager daemon y 2024-06-18T02:42:15.202 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-mon[29791]: osdmap e60: 8 total, 8 up, 8 in 2024-06-18T02:42:15.202 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-mon[25094]: Active manager daemon y restarted 2024-06-18T02:42:15.203 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-mon[25094]: Activating manager daemon y 2024-06-18T02:42:15.203 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-mon[25094]: osdmap e60: 8 total, 8 up, 8 in 2024-06-18T02:42:15.203 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:14 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:14.941+0000 7f2f8e885000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T02:42:15.203 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:42:15] ENGINE Bus STARTING 2024-06-18T02:42:15.203 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: CherryPy Checker: 2024-06-18T02:42:15.203 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: The Application mounted at '' has an empty config. 2024-06-18T02:42:15.203 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:42:15.203 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:42:15] ENGINE Serving on http://:::9283 2024-06-18T02:42:15.203 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:42:15] ENGINE Bus STARTED 2024-06-18T02:42:15.297 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:15 smithi186 ceph-mon[25944]: Active manager daemon y restarted 2024-06-18T02:42:15.297 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:15 smithi186 ceph-mon[25944]: Activating manager daemon y 2024-06-18T02:42:15.297 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:15 smithi186 ceph-mon[25944]: osdmap e60: 8 total, 8 up, 8 in 2024-06-18T02:42:15.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:15.015+0000 7f1fe99ad000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T02:42:15.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: [18/Jun/2024:02:42:15] ENGINE Bus STARTING 2024-06-18T02:42:15.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: CherryPy Checker: 2024-06-18T02:42:15.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: The Application mounted at '' has an empty config. 2024-06-18T02:42:15.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: 2024-06-18T02:42:15.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: [18/Jun/2024:02:42:15] ENGINE Serving on http://:::9283 2024-06-18T02:42:15.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: [18/Jun/2024:02:42:15] ENGINE Bus STARTED 2024-06-18T02:42:15.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:42:15] ENGINE Bus STARTING 2024-06-18T02:42:15.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:42:15] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T02:42:15.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:42:15] ENGINE Bus STARTED 2024-06-18T02:42:16.028 INFO:teuthology.orchestra.run.smithi186.stdout:Scheduled node-exporter update... 2024-06-18T02:42:16.042 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: mgrmap e17: y(active, starting, since 0.0589198s), standbys: x 2024-06-18T02:42:16.042 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:42:16.042 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:42:16.042 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:42:16.042 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T02:42:16.042 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T02:42:16.042 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:42:16.042 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:42:16.042 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:42:16.042 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:42:16.043 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:42:16.043 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:42:16.043 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:42:16.043 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:42:16.043 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T02:42:16.043 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T02:42:16.043 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T02:42:16.043 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: Manager daemon y is now available 2024-06-18T02:42:16.043 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: Standby manager daemon x restarted 2024-06-18T02:42:16.043 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: Standby manager daemon x started 2024-06-18T02:42:16.043 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/3441275931' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T02:42:16.043 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/3441275931' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:42:16.043 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/3441275931' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T02:42:16.044 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/3441275931' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:42:16.044 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:42:16.044 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:42:16.044 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:42:16.044 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:42:16.044 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:16.044 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:42:16.044 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:42:16.044 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:42:16.044 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:42:16.044 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:16 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:16.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: mgrmap e17: y(active, starting, since 0.0589198s), standbys: x 2024-06-18T02:42:16.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:42:16.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:42:16.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:42:16.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T02:42:16.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T02:42:16.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:42:16.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:42:16.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:42:16.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:42:16.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: Manager daemon y is now available 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: Standby manager daemon x restarted 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: Standby manager daemon x started 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/3441275931' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/3441275931' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/3441275931' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/3441275931' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:42:16.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:42:16.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:42:16.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:42:16.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: mgrmap e17: y(active, starting, since 0.0589198s), standbys: x 2024-06-18T02:42:16.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:42:16.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:42:16.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:42:16.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T02:42:16.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T02:42:16.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:42:16.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:42:16.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: Manager daemon y is now available 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: Standby manager daemon x restarted 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: Standby manager daemon x started 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/3441275931' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/3441275931' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/3441275931' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/3441275931' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:42:16.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:42:16.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:42:16.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:42:16.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:42:16.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:16.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:42:16.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:42:16.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:42:16.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:42:16.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:16.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:42:16.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:16.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:42:16.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:42:16.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:42:16.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:42:16.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:16 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:16.828 DEBUG:teuthology.orchestra.run.smithi163:node-exporter.a> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.a.service 2024-06-18T02:42:16.831 DEBUG:teuthology.orchestra.run.smithi186:node-exporter.b> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.b.service 2024-06-18T02:42:16.833 INFO:tasks.cephadm:Adding alertmanager.a on smithi163 2024-06-18T02:42:16.833 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch apply alertmanager '1;smithi163=a' 2024-06-18T02:42:17.052 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:17 smithi186 ceph-mon[25944]: [18/Jun/2024:02:42:15] ENGINE Bus STARTING 2024-06-18T02:42:17.052 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:17 smithi186 ceph-mon[25944]: [18/Jun/2024:02:42:15] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T02:42:17.052 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:17 smithi186 ceph-mon[25944]: [18/Jun/2024:02:42:15] ENGINE Bus STARTED 2024-06-18T02:42:17.052 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:17 smithi186 ceph-mon[25944]: mgrmap e18: y(active, since 1.07491s), standbys: x 2024-06-18T02:42:17.052 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:17 smithi186 ceph-mon[25944]: from='client.24335 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi163=a;smithi186=b", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:42:17.052 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:17 smithi186 ceph-mon[25944]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:17.052 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:17 smithi186 ceph-mon[25944]: Saving service node-exporter spec with placement smithi163=a;smithi186=b;count:2 2024-06-18T02:42:17.052 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:17 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:17.053 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:17 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:17.053 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:17 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:17.053 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:17 smithi186 ceph-mon[25944]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T02:42:17.053 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:17 smithi186 ceph-mon[25944]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:17.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[29791]: [18/Jun/2024:02:42:15] ENGINE Bus STARTING 2024-06-18T02:42:17.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[29791]: [18/Jun/2024:02:42:15] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T02:42:17.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[29791]: [18/Jun/2024:02:42:15] ENGINE Bus STARTED 2024-06-18T02:42:17.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[29791]: mgrmap e18: y(active, since 1.07491s), standbys: x 2024-06-18T02:42:17.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[29791]: from='client.24335 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi163=a;smithi186=b", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:42:17.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[29791]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:17.226 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[29791]: Saving service node-exporter spec with placement smithi163=a;smithi186=b;count:2 2024-06-18T02:42:17.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:17.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:17.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:17.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[29791]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T02:42:17.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[29791]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:17.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[25094]: [18/Jun/2024:02:42:15] ENGINE Bus STARTING 2024-06-18T02:42:17.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[25094]: [18/Jun/2024:02:42:15] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T02:42:17.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[25094]: [18/Jun/2024:02:42:15] ENGINE Bus STARTED 2024-06-18T02:42:17.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[25094]: mgrmap e18: y(active, since 1.07491s), standbys: x 2024-06-18T02:42:17.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[25094]: from='client.24335 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi163=a;smithi186=b", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:42:17.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[25094]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:17.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[25094]: Saving service node-exporter spec with placement smithi163=a;smithi186=b;count:2 2024-06-18T02:42:17.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:17.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:17.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:17.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[25094]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T02:42:17.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:17 smithi163 ceph-mon[25094]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:18.093 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:18 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:18.093 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:18 smithi186 ceph-mon[25944]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:42:18.093 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:18 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:18.093 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:18 smithi186 ceph-mon[25944]: mgrmap e19: y(active, since 2s), standbys: x 2024-06-18T02:42:18.093 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:18 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:18.093 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:18 smithi186 ceph-mon[25944]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T02:42:18.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:18 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:18.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:18 smithi163 ceph-mon[25094]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:42:18.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:18 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:18.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:18 smithi163 ceph-mon[25094]: mgrmap e19: y(active, since 2s), standbys: x 2024-06-18T02:42:18.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:18 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:18.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:18 smithi163 ceph-mon[25094]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T02:42:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:18 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:18 smithi163 ceph-mon[29791]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:42:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:18 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:18 smithi163 ceph-mon[29791]: mgrmap e19: y(active, since 2s), standbys: x 2024-06-18T02:42:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:18 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:18 smithi163 ceph-mon[29791]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T02:42:19.056 INFO:teuthology.orchestra.run.smithi186.stdout:Scheduled alertmanager update... 2024-06-18T02:42:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:19 smithi186 ceph-mon[25944]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:42:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:19 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:19 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:19 smithi186 ceph-mon[25944]: Deploying daemon prometheus.a on smithi186 2024-06-18T02:42:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:19 smithi186 ceph-mon[25944]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:19 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:19.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:19 smithi163 ceph-mon[29791]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:42:19.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:19 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:19.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:19 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:19.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:19 smithi163 ceph-mon[29791]: Deploying daemon prometheus.a on smithi186 2024-06-18T02:42:19.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:19 smithi163 ceph-mon[29791]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:19.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:19 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:19.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:19 smithi163 ceph-mon[25094]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:42:19.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:19 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:19.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:19 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:19.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:19 smithi163 ceph-mon[25094]: Deploying daemon prometheus.a on smithi186 2024-06-18T02:42:19.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:19 smithi163 ceph-mon[25094]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:19.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:19 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:19.783 DEBUG:teuthology.orchestra.run.smithi163:alertmanager.a> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@alertmanager.a.service 2024-06-18T02:42:19.786 INFO:tasks.cephadm:Adding grafana.a on smithi186 2024-06-18T02:42:19.786 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph orch apply grafana '1;smithi186=a' 2024-06-18T02:42:20.337 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:20 smithi186 ceph-mon[25944]: from='client.24353 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi163=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:42:20.337 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:20 smithi186 ceph-mon[25944]: Saving service alertmanager spec with placement smithi163=a;count:1 2024-06-18T02:42:20.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:20 smithi163 ceph-mon[29791]: from='client.24353 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi163=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:42:20.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:20 smithi163 ceph-mon[29791]: Saving service alertmanager spec with placement smithi163=a;count:1 2024-06-18T02:42:20.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:20 smithi163 ceph-mon[25094]: from='client.24353 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi163=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:42:20.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:20 smithi163 ceph-mon[25094]: Saving service alertmanager spec with placement smithi163=a;count:1 2024-06-18T02:42:21.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:21 smithi163 ceph-mon[29791]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:21.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:21 smithi163 ceph-mon[25094]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:21.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:21 smithi186 ceph-mon[25944]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:22.598 INFO:teuthology.orchestra.run.smithi186.stdout:Scheduled grafana update... 2024-06-18T02:42:23.553 DEBUG:teuthology.orchestra.run.smithi186:grafana.a> sudo journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@grafana.a.service 2024-06-18T02:42:23.556 INFO:tasks.cephadm:Setting up client nodes... 2024-06-18T02:42:23.556 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-06-18T02:42:23.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:23 smithi186 ceph-mon[25944]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi186=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:42:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:23 smithi186 ceph-mon[25944]: Saving service grafana spec with placement smithi186=a;count:1 2024-06-18T02:42:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:23 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:23 smithi186 ceph-mon[25944]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:23.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:23 smithi163 ceph-mon[25094]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi186=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:42:23.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:23 smithi163 ceph-mon[25094]: Saving service grafana spec with placement smithi186=a;count:1 2024-06-18T02:42:23.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:23 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:23.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:23 smithi163 ceph-mon[25094]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:23.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:23 smithi163 ceph-mon[29791]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi186=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:42:23.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:23 smithi163 ceph-mon[29791]: Saving service grafana spec with placement smithi186=a;count:1 2024-06-18T02:42:23.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:23 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:23.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:23 smithi163 ceph-mon[29791]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:25.196 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:25 smithi163 ceph-mon[25094]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:25.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:25 smithi163 ceph-mon[29791]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:25.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:25 smithi186 ceph-mon[25944]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:25.678 INFO:teuthology.orchestra.run.smithi163.stdout:[client.0] 2024-06-18T02:42:25.678 INFO:teuthology.orchestra.run.smithi163.stdout: key = AQAR9HBmtKU4KBAAYZUAbPdHAPIozZxFoMLiTA== 2024-06-18T02:42:26.289 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2024-06-18T02:42:26.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-06-18T02:42:26.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-06-18T02:42:26.328 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-06-18T02:42:26.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:26 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/4153615630' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-18T02:42:26.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:26 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-18T02:42:26.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:26 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-18T02:42:26.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:26 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/4153615630' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-18T02:42:26.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:26 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-18T02:42:26.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:26 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-18T02:42:26.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:26 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/4153615630' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-18T02:42:26.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:26 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-18T02:42:26.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:26 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-18T02:42:27.408 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:27 smithi186 ceph-mon[25944]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:27.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:27 smithi163 ceph-mon[29791]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:27.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:27 smithi163 ceph-mon[25094]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:29.143 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:29 smithi186 ceph-mon[25944]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:29.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:29 smithi163 ceph-mon[29791]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:29.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:29 smithi163 ceph-mon[25094]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:29.623 INFO:teuthology.orchestra.run.smithi186.stdout:[client.1] 2024-06-18T02:42:29.624 INFO:teuthology.orchestra.run.smithi186.stdout: key = AQAV9HBm2Ir2JBAASD5Hga/fmbOYT3dOM2GjAQ== 2024-06-18T02:42:30.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:30 smithi186 ceph-mon[25944]: from='client.? 172.21.15.186:0/1730640897' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-18T02:42:30.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:30 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-18T02:42:30.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:30 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-18T02:42:30.373 DEBUG:teuthology.orchestra.run.smithi186:> set -ex 2024-06-18T02:42:30.373 DEBUG:teuthology.orchestra.run.smithi186:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-06-18T02:42:30.373 DEBUG:teuthology.orchestra.run.smithi186:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-06-18T02:42:30.411 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-06-18T02:42:30.411 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-06-18T02:42:30.412 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph mgr dump --format=json 2024-06-18T02:42:30.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:30 smithi163 ceph-mon[25094]: from='client.? 172.21.15.186:0/1730640897' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-18T02:42:30.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:30 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-18T02:42:30.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:30 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-18T02:42:30.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:30 smithi163 ceph-mon[29791]: from='client.? 172.21.15.186:0/1730640897' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-18T02:42:30.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:30 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-18T02:42:30.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:30 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-18T02:42:30.632 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:31.530 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:31 smithi186 ceph-mon[25944]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:31.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:31 smithi163 ceph-mon[25094]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:31.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:31 smithi163 ceph-mon[29791]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:32.215 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:31 smithi186 systemd[1]: Starting Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:42:32.449 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:42:32.468 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 podman[40964]: 2024-06-18 02:42:32.118949564 +0000 UTC m=+0.018904060 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-06-18T02:42:32.468 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 podman[40964]: 2024-06-18 02:42:32.216007574 +0000 UTC m=+0.115962056 container create cdd53f400d156cbb0c10d7c76a0992c9b90187395815bf77ee1b14f9012314fc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:42:32.468 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 podman[40964]: 2024-06-18 02:42:32.468725287 +0000 UTC m=+0.368679775 container init cdd53f400d156cbb0c10d7c76a0992c9b90187395815bf77ee1b14f9012314fc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:42:32.621 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:32 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1477718045' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-18T02:42:32.641 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:32 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1477718045' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-18T02:42:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1477718045' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-18T02:42:32.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 podman[40964]: 2024-06-18 02:42:32.47178446 +0000 UTC m=+0.371738951 container start cdd53f400d156cbb0c10d7c76a0992c9b90187395815bf77ee1b14f9012314fc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:42:32.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.495Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-06-18T02:42:32.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.495Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-06-18T02:42:32.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.495Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-06-18T02:42:32.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.495Z caller=main.go:518 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi186 (none))" 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.495Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.495Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.497Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.497Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.498Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.501Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.501Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.641µs 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.501Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.501Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.501Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.07µs wal_replay_duration=229.115µs total_replay_duration=281.058µs 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.502Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.502Z caller=main.go:947 level=info msg="TSDB started" 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.502Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.517Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.544331ms db_storage=737ns remote_storage=1.707µs web_handler=394ns query_engine=868ns scrape=691.586µs scrape_sd=18.478µs notify=840ns notify_sd=2.003µs rules=14.617286ms 2024-06-18T02:42:32.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:42:32.517Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-06-18T02:42:32.820 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 bash[40964]: cdd53f400d156cbb0c10d7c76a0992c9b90187395815bf77ee1b14f9012314fc 2024-06-18T02:42:32.820 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:42:32 smithi186 systemd[1]: Started Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:42:33.111 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":19,"active_gid":24343,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6800","nonce":3786129084},{"type":"v1","addr":"172.21.15.163:6801","nonce":3786129084}]},"active_addr":"172.21.15.163:6801/3786129084","active_change":"2024-06-18T02:42:14.944776+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24323,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":3854405794}]},{"addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":1481594701}]},{"addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":4111534652}]},{"addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":3601932060}]}]}} 2024-06-18T02:42:33.115 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-06-18T02:42:33.115 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-06-18T02:42:33.116 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd dump --format=json 2024-06-18T02:42:33.402 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:33 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:33 smithi163 ceph-mon[25094]: Deploying daemon node-exporter.a on smithi163 2024-06-18T02:42:33.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:33 smithi163 ceph-mon[25094]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:33.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:33 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:33.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:33 smithi163 ceph-mon[29791]: Deploying daemon node-exporter.a on smithi163 2024-06-18T02:42:33.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:33 smithi163 ceph-mon[29791]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:33 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:33 smithi186 ceph-mon[25944]: Deploying daemon node-exporter.a on smithi163 2024-06-18T02:42:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:33 smithi186 ceph-mon[25944]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:35.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:35 smithi186 ceph-mon[25944]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:35.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:35 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:35.351 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:42:35.352 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":60,"fsid":"929affee-2d1b-11ef-bc9d-c7b262605968","created":"2024-06-18T02:37:00.382705+0000","modified":"2024-06-18T02:42:14.944446+0000","last_up_change":"2024-06-18T02:41:56.477912+0000","last_in_change":"2024-06-18T02:41:39.027963+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-06-18T02:40:00.660490+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"142df380-e27f-4351-b05c-367d07947ced","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":4251995344},{"type":"v1","addr":"172.21.15.163:6803","nonce":4251995344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":4251995344},{"type":"v1","addr":"172.21.15.163:6805","nonce":4251995344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":4251995344},{"type":"v1","addr":"172.21.15.163:6809","nonce":4251995344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":4251995344},{"type":"v1","addr":"172.21.15.163:6807","nonce":4251995344}]},"public_addr":"172.21.15.163:6803/4251995344","cluster_addr":"172.21.15.163:6805/4251995344","heartbeat_back_addr":"172.21.15.163:6809/4251995344","heartbeat_front_addr":"172.21.15.163:6807/4251995344","state":["exists","up"]},{"osd":1,"uuid":"696b47f4-f00e-4c37-80ec-6df94769a34f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":1462975684},{"type":"v1","addr":"172.21.15.163:6811","nonce":1462975684}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":1462975684},{"type":"v1","addr":"172.21.15.163:6813","nonce":1462975684}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":1462975684},{"type":"v1","addr":"172.21.15.163:6817","nonce":1462975684}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":1462975684},{"type":"v1","addr":"172.21.15.163:6815","nonce":1462975684}]},"public_addr":"172.21.15.163:6811/1462975684","cluster_addr":"172.21.15.163:6813/1462975684","heartbeat_back_addr":"172.21.15.163:6817/1462975684","heartbeat_front_addr":"172.21.15.163:6815/1462975684","state":["exists","up"]},{"osd":2,"uuid":"ad504a3c-fa5c-4b4c-a079-a66b52088d68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":923602700},{"type":"v1","addr":"172.21.15.163:6819","nonce":923602700}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":923602700},{"type":"v1","addr":"172.21.15.163:6821","nonce":923602700}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":923602700},{"type":"v1","addr":"172.21.15.163:6825","nonce":923602700}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":923602700},{"type":"v1","addr":"172.21.15.163:6823","nonce":923602700}]},"public_addr":"172.21.15.163:6819/923602700","cluster_addr":"172.21.15.163:6821/923602700","heartbeat_back_addr":"172.21.15.163:6825/923602700","heartbeat_front_addr":"172.21.15.163:6823/923602700","state":["exists","up"]},{"osd":3,"uuid":"f9f9da4f-4afc-48cf-92a2-f714ca3f4c4f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":2061575135},{"type":"v1","addr":"172.21.15.163:6827","nonce":2061575135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":2061575135},{"type":"v1","addr":"172.21.15.163:6829","nonce":2061575135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":2061575135},{"type":"v1","addr":"172.21.15.163:6833","nonce":2061575135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":2061575135},{"type":"v1","addr":"172.21.15.163:6831","nonce":2061575135}]},"public_addr":"172.21.15.163:6827/2061575135","cluster_addr":"172.21.15.163:6829/2061575135","heartbeat_back_addr":"172.21.15.163:6833/2061575135","heartbeat_front_addr":"172.21.15.163:6831/2061575135","state":["exists","up"]},{"osd":4,"uuid":"12a9d06c-9aab-4659-8533-f57336078d0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6800","nonce":368766852},{"type":"v1","addr":"172.21.15.186:6801","nonce":368766852}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6802","nonce":368766852},{"type":"v1","addr":"172.21.15.186:6803","nonce":368766852}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6806","nonce":368766852},{"type":"v1","addr":"172.21.15.186:6807","nonce":368766852}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6804","nonce":368766852},{"type":"v1","addr":"172.21.15.186:6805","nonce":368766852}]},"public_addr":"172.21.15.186:6801/368766852","cluster_addr":"172.21.15.186:6803/368766852","heartbeat_back_addr":"172.21.15.186:6807/368766852","heartbeat_front_addr":"172.21.15.186:6805/368766852","state":["exists","up"]},{"osd":5,"uuid":"9bb0e529-2a21-425f-9da9-aa81ae22a52c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6808","nonce":1758819799},{"type":"v1","addr":"172.21.15.186:6809","nonce":1758819799}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6810","nonce":1758819799},{"type":"v1","addr":"172.21.15.186:6811","nonce":1758819799}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6814","nonce":1758819799},{"type":"v1","addr":"172.21.15.186:6815","nonce":1758819799}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6812","nonce":1758819799},{"type":"v1","addr":"172.21.15.186:6813","nonce":1758819799}]},"public_addr":"172.21.15.186:6809/1758819799","cluster_addr":"172.21.15.186:6811/1758819799","heartbeat_back_addr":"172.21.15.186:6815/1758819799","heartbeat_front_addr":"172.21.15.186:6813/1758819799","state":["exists","up"]},{"osd":6,"uuid":"002ed2bc-653a-475e-9905-cd5e23dd10c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6816","nonce":3657102327},{"type":"v1","addr":"172.21.15.186:6817","nonce":3657102327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6818","nonce":3657102327},{"type":"v1","addr":"172.21.15.186:6819","nonce":3657102327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6822","nonce":3657102327},{"type":"v1","addr":"172.21.15.186:6823","nonce":3657102327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6820","nonce":3657102327},{"type":"v1","addr":"172.21.15.186:6821","nonce":3657102327}]},"public_addr":"172.21.15.186:6817/3657102327","cluster_addr":"172.21.15.186:6819/3657102327","heartbeat_back_addr":"172.21.15.186:6823/3657102327","heartbeat_front_addr":"172.21.15.186:6821/3657102327","state":["exists","up"]},{"osd":7,"uuid":"d79629d8-851c-4e0d-b1da-752a6505037a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6824","nonce":894630858},{"type":"v1","addr":"172.21.15.186:6825","nonce":894630858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6826","nonce":894630858},{"type":"v1","addr":"172.21.15.186:6827","nonce":894630858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6830","nonce":894630858},{"type":"v1","addr":"172.21.15.186:6831","nonce":894630858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6828","nonce":894630858},{"type":"v1","addr":"172.21.15.186:6829","nonce":894630858}]},"public_addr":"172.21.15.186:6825/894630858","cluster_addr":"172.21.15.186:6827/894630858","heartbeat_back_addr":"172.21.15.186:6831/894630858","heartbeat_front_addr":"172.21.15.186:6829/894630858","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:39:11.602779+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:39:34.500246+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:39:57.071084+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:40:21.371956+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:40:43.043329+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:41:06.426162+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:41:30.195208+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:41:54.520226+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/1760538181":"2024-06-19T02:42:14.944403+0000","172.21.15.163:0/1226154455":"2024-06-19T02:42:14.944403+0000","172.21.15.163:0/3210434964":"2024-06-19T02:42:14.944403+0000","172.21.15.163:0/2324578116":"2024-06-19T02:42:14.944403+0000","172.21.15.163:6801/911896598":"2024-06-19T02:42:14.944403+0000","172.21.15.163:6800/911896598":"2024-06-19T02:42:14.944403+0000","172.21.15.163:0/1291328983":"2024-06-19T02:37:54.311318+0000","172.21.15.163:6801/4217980061":"2024-06-19T02:37:54.311318+0000","172.21.15.163:0/3911242098":"2024-06-19T02:37:54.311318+0000","172.21.15.163:6800/396937736":"2024-06-19T02:37:32.086178+0000","172.21.15.163:6801/396937736":"2024-06-19T02:37:32.086178+0000","172.21.15.163:0/921115307":"2024-06-19T02:37:54.311318+0000","172.21.15.163:0/3175087117":"2024-06-19T02:37:32.086178+0000","172.21.15.163:0/2926912200":"2024-06-19T02:37:32.086178+0000","172.21.15.163:6800/4217980061":"2024-06-19T02:37:54.311318+0000","172.21.15.163:0/538097455":"2024-06-19T02:37:32.086178+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-06-18T02:42:35.352 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:35 smithi163 ceph-mon[25094]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:35.352 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:35 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:35.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:35 smithi163 ceph-mon[29791]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:35.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:35 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:36.534 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:36 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3590283334' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-18T02:42:36.535 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:36 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3590283334' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-18T02:42:36.535 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:36 smithi163 bash[44665]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-06-18T02:42:36.535 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:36 smithi163 bash[44665]: Writing manifest to image destination 2024-06-18T02:42:36.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:36 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3590283334' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-18T02:42:36.792 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:36 smithi163 podman[44665]: 2024-06-18 02:42:36.740973247 +0000 UTC m=+2.683233436 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-06-18T02:42:36.864 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-06-18T02:42:36.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd dump --format=json 2024-06-18T02:42:37.143 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:36 smithi163 podman[44665]: 2024-06-18 02:42:36.949707703 +0000 UTC m=+2.891967883 container create 4ed4364c9a98af4f1c183d062706b0767a85d75beccd2605be9b27bb00fa47f7 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T02:42:37.143 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 podman[44665]: 2024-06-18 02:42:37.144394884 +0000 UTC m=+3.086655061 container init 4ed4364c9a98af4f1c183d062706b0767a85d75beccd2605be9b27bb00fa47f7 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T02:42:37.264 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:37.484 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-mon[29791]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:37.485 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-mon[25094]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:37.485 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 podman[44665]: 2024-06-18 02:42:37.147310273 +0000 UTC m=+3.089570451 container start 4ed4364c9a98af4f1c183d062706b0767a85d75beccd2605be9b27bb00fa47f7 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T02:42:37.485 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.151Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-06-18T02:42:37.485 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.151Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-06-18T02:42:37.485 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.151Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-06-18T02:42:37.486 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.151Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-06-18T02:42:37.486 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-06-18T02:42:37.486 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=arp 2024-06-18T02:42:37.486 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=bcache 2024-06-18T02:42:37.486 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=bonding 2024-06-18T02:42:37.486 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=btrfs 2024-06-18T02:42:37.486 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=conntrack 2024-06-18T02:42:37.486 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=cpu 2024-06-18T02:42:37.486 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-06-18T02:42:37.486 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=diskstats 2024-06-18T02:42:37.486 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=dmi 2024-06-18T02:42:37.486 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=edac 2024-06-18T02:42:37.486 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=entropy 2024-06-18T02:42:37.487 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-06-18T02:42:37.487 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=filefd 2024-06-18T02:42:37.487 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=filesystem 2024-06-18T02:42:37.487 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=hwmon 2024-06-18T02:42:37.487 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=infiniband 2024-06-18T02:42:37.487 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=ipvs 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=loadavg 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=mdadm 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=meminfo 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=netclass 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=netdev 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=netstat 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=nfs 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=nfsd 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=nvme 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=os 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=pressure 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=rapl 2024-06-18T02:42:37.488 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=schedstat 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=sockstat 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=softnet 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=stat 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=tapestats 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=textfile 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=time 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=uname 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=vmstat 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=xfs 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:115 level=info collector=zfs 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-06-18T02:42:37.489 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[44814]: ts=2024-06-18T02:42:37.152Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-06-18T02:42:37.490 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 bash[44665]: 4ed4364c9a98af4f1c183d062706b0767a85d75beccd2605be9b27bb00fa47f7 2024-06-18T02:42:37.490 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:42:37 smithi163 systemd[1]: Started Ceph node-exporter.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:42:37.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:37 smithi186 ceph-mon[25944]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:38.297 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:38 smithi186 bash[41195]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-06-18T02:42:38.417 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:38 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:38.417 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:38 smithi163 ceph-mon[29791]: Deploying daemon node-exporter.b on smithi186 2024-06-18T02:42:38.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:38 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:38.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:38 smithi163 ceph-mon[25094]: Deploying daemon node-exporter.b on smithi186 2024-06-18T02:42:38.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:38 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:38.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:38 smithi186 ceph-mon[25944]: Deploying daemon node-exporter.b on smithi186 2024-06-18T02:42:38.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:38 smithi186 bash[41195]: Getting image source signatures 2024-06-18T02:42:38.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:38 smithi186 bash[41195]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-06-18T02:42:38.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:38 smithi186 bash[41195]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-06-18T02:42:38.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:38 smithi186 bash[41195]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-06-18T02:42:38.744 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:38 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:42:38] "GET /metrics HTTP/1.1" 200 191520 "" "Prometheus/2.33.4" 2024-06-18T02:42:39.170 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:42:39.170 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":60,"fsid":"929affee-2d1b-11ef-bc9d-c7b262605968","created":"2024-06-18T02:37:00.382705+0000","modified":"2024-06-18T02:42:14.944446+0000","last_up_change":"2024-06-18T02:41:56.477912+0000","last_in_change":"2024-06-18T02:41:39.027963+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-06-18T02:40:00.660490+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"142df380-e27f-4351-b05c-367d07947ced","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":4251995344},{"type":"v1","addr":"172.21.15.163:6803","nonce":4251995344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":4251995344},{"type":"v1","addr":"172.21.15.163:6805","nonce":4251995344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":4251995344},{"type":"v1","addr":"172.21.15.163:6809","nonce":4251995344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":4251995344},{"type":"v1","addr":"172.21.15.163:6807","nonce":4251995344}]},"public_addr":"172.21.15.163:6803/4251995344","cluster_addr":"172.21.15.163:6805/4251995344","heartbeat_back_addr":"172.21.15.163:6809/4251995344","heartbeat_front_addr":"172.21.15.163:6807/4251995344","state":["exists","up"]},{"osd":1,"uuid":"696b47f4-f00e-4c37-80ec-6df94769a34f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":1462975684},{"type":"v1","addr":"172.21.15.163:6811","nonce":1462975684}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":1462975684},{"type":"v1","addr":"172.21.15.163:6813","nonce":1462975684}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":1462975684},{"type":"v1","addr":"172.21.15.163:6817","nonce":1462975684}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":1462975684},{"type":"v1","addr":"172.21.15.163:6815","nonce":1462975684}]},"public_addr":"172.21.15.163:6811/1462975684","cluster_addr":"172.21.15.163:6813/1462975684","heartbeat_back_addr":"172.21.15.163:6817/1462975684","heartbeat_front_addr":"172.21.15.163:6815/1462975684","state":["exists","up"]},{"osd":2,"uuid":"ad504a3c-fa5c-4b4c-a079-a66b52088d68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":923602700},{"type":"v1","addr":"172.21.15.163:6819","nonce":923602700}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":923602700},{"type":"v1","addr":"172.21.15.163:6821","nonce":923602700}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":923602700},{"type":"v1","addr":"172.21.15.163:6825","nonce":923602700}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":923602700},{"type":"v1","addr":"172.21.15.163:6823","nonce":923602700}]},"public_addr":"172.21.15.163:6819/923602700","cluster_addr":"172.21.15.163:6821/923602700","heartbeat_back_addr":"172.21.15.163:6825/923602700","heartbeat_front_addr":"172.21.15.163:6823/923602700","state":["exists","up"]},{"osd":3,"uuid":"f9f9da4f-4afc-48cf-92a2-f714ca3f4c4f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":2061575135},{"type":"v1","addr":"172.21.15.163:6827","nonce":2061575135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":2061575135},{"type":"v1","addr":"172.21.15.163:6829","nonce":2061575135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":2061575135},{"type":"v1","addr":"172.21.15.163:6833","nonce":2061575135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":2061575135},{"type":"v1","addr":"172.21.15.163:6831","nonce":2061575135}]},"public_addr":"172.21.15.163:6827/2061575135","cluster_addr":"172.21.15.163:6829/2061575135","heartbeat_back_addr":"172.21.15.163:6833/2061575135","heartbeat_front_addr":"172.21.15.163:6831/2061575135","state":["exists","up"]},{"osd":4,"uuid":"12a9d06c-9aab-4659-8533-f57336078d0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6800","nonce":368766852},{"type":"v1","addr":"172.21.15.186:6801","nonce":368766852}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6802","nonce":368766852},{"type":"v1","addr":"172.21.15.186:6803","nonce":368766852}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6806","nonce":368766852},{"type":"v1","addr":"172.21.15.186:6807","nonce":368766852}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6804","nonce":368766852},{"type":"v1","addr":"172.21.15.186:6805","nonce":368766852}]},"public_addr":"172.21.15.186:6801/368766852","cluster_addr":"172.21.15.186:6803/368766852","heartbeat_back_addr":"172.21.15.186:6807/368766852","heartbeat_front_addr":"172.21.15.186:6805/368766852","state":["exists","up"]},{"osd":5,"uuid":"9bb0e529-2a21-425f-9da9-aa81ae22a52c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6808","nonce":1758819799},{"type":"v1","addr":"172.21.15.186:6809","nonce":1758819799}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6810","nonce":1758819799},{"type":"v1","addr":"172.21.15.186:6811","nonce":1758819799}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6814","nonce":1758819799},{"type":"v1","addr":"172.21.15.186:6815","nonce":1758819799}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6812","nonce":1758819799},{"type":"v1","addr":"172.21.15.186:6813","nonce":1758819799}]},"public_addr":"172.21.15.186:6809/1758819799","cluster_addr":"172.21.15.186:6811/1758819799","heartbeat_back_addr":"172.21.15.186:6815/1758819799","heartbeat_front_addr":"172.21.15.186:6813/1758819799","state":["exists","up"]},{"osd":6,"uuid":"002ed2bc-653a-475e-9905-cd5e23dd10c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6816","nonce":3657102327},{"type":"v1","addr":"172.21.15.186:6817","nonce":3657102327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6818","nonce":3657102327},{"type":"v1","addr":"172.21.15.186:6819","nonce":3657102327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6822","nonce":3657102327},{"type":"v1","addr":"172.21.15.186:6823","nonce":3657102327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6820","nonce":3657102327},{"type":"v1","addr":"172.21.15.186:6821","nonce":3657102327}]},"public_addr":"172.21.15.186:6817/3657102327","cluster_addr":"172.21.15.186:6819/3657102327","heartbeat_back_addr":"172.21.15.186:6823/3657102327","heartbeat_front_addr":"172.21.15.186:6821/3657102327","state":["exists","up"]},{"osd":7,"uuid":"d79629d8-851c-4e0d-b1da-752a6505037a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6824","nonce":894630858},{"type":"v1","addr":"172.21.15.186:6825","nonce":894630858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6826","nonce":894630858},{"type":"v1","addr":"172.21.15.186:6827","nonce":894630858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6830","nonce":894630858},{"type":"v1","addr":"172.21.15.186:6831","nonce":894630858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.186:6828","nonce":894630858},{"type":"v1","addr":"172.21.15.186:6829","nonce":894630858}]},"public_addr":"172.21.15.186:6825/894630858","cluster_addr":"172.21.15.186:6827/894630858","heartbeat_back_addr":"172.21.15.186:6831/894630858","heartbeat_front_addr":"172.21.15.186:6829/894630858","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:39:11.602779+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:39:34.500246+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:39:57.071084+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:40:21.371956+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:40:43.043329+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:41:06.426162+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:41:30.195208+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-18T02:41:54.520226+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/1760538181":"2024-06-19T02:42:14.944403+0000","172.21.15.163:0/1226154455":"2024-06-19T02:42:14.944403+0000","172.21.15.163:0/3210434964":"2024-06-19T02:42:14.944403+0000","172.21.15.163:0/2324578116":"2024-06-19T02:42:14.944403+0000","172.21.15.163:6801/911896598":"2024-06-19T02:42:14.944403+0000","172.21.15.163:6800/911896598":"2024-06-19T02:42:14.944403+0000","172.21.15.163:0/1291328983":"2024-06-19T02:37:54.311318+0000","172.21.15.163:6801/4217980061":"2024-06-19T02:37:54.311318+0000","172.21.15.163:0/3911242098":"2024-06-19T02:37:54.311318+0000","172.21.15.163:6800/396937736":"2024-06-19T02:37:32.086178+0000","172.21.15.163:6801/396937736":"2024-06-19T02:37:32.086178+0000","172.21.15.163:0/921115307":"2024-06-19T02:37:54.311318+0000","172.21.15.163:0/3175087117":"2024-06-19T02:37:32.086178+0000","172.21.15.163:0/2926912200":"2024-06-19T02:37:32.086178+0000","172.21.15.163:6800/4217980061":"2024-06-19T02:37:54.311318+0000","172.21.15.163:0/538097455":"2024-06-19T02:37:32.086178+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-06-18T02:42:39.438 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:39 smithi163 ceph-mon[29791]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:39.438 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:39 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3386670185' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-18T02:42:39.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:39 smithi163 ceph-mon[25094]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:39.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:39 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3386670185' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-18T02:42:39.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:39 smithi186 ceph-mon[25944]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:39.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:39 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3386670185' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-18T02:42:40.053 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-06-18T02:42:40.053 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-06-18T02:42:40.054 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-06-18T02:42:40.054 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-06-18T02:42:40.055 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-06-18T02:42:40.055 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-06-18T02:42:40.055 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-06-18T02:42:40.056 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-06-18T02:42:40.317 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:39 smithi186 bash[41195]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-06-18T02:42:40.318 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 bash[41195]: Writing manifest to image destination 2024-06-18T02:42:40.495 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:40.496 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:40.498 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:40.498 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:40.499 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:40.499 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:40.508 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:40.509 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:40.702 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 podman[41195]: 2024-06-18 02:42:40.378127197 +0000 UTC m=+2.320351278 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-06-18T02:42:40.703 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 podman[41195]: 2024-06-18 02:42:40.485136219 +0000 UTC m=+2.427360342 container create 48a19a2881139dae70078a2e6156c9701734fb1c32859d415a7e3699ea00628f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T02:42:41.018 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 podman[41195]: 2024-06-18 02:42:40.703754364 +0000 UTC m=+2.645978457 container init 48a19a2881139dae70078a2e6156c9701734fb1c32859d415a7e3699ea00628f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T02:42:41.018 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 podman[41195]: 2024-06-18 02:42:40.706772237 +0000 UTC m=+2.648996323 container start 48a19a2881139dae70078a2e6156c9701734fb1c32859d415a7e3699ea00628f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T02:42:41.018 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.710Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-06-18T02:42:41.018 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.710Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-06-18T02:42:41.019 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-06-18T02:42:41.019 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-06-18T02:42:41.019 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-06-18T02:42:41.019 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=arp 2024-06-18T02:42:41.019 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=bcache 2024-06-18T02:42:41.019 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=bonding 2024-06-18T02:42:41.019 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=btrfs 2024-06-18T02:42:41.019 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=conntrack 2024-06-18T02:42:41.019 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=cpu 2024-06-18T02:42:41.019 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-06-18T02:42:41.019 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=diskstats 2024-06-18T02:42:41.019 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=dmi 2024-06-18T02:42:41.019 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=edac 2024-06-18T02:42:41.020 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=entropy 2024-06-18T02:42:41.020 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-06-18T02:42:41.020 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=filefd 2024-06-18T02:42:41.020 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=filesystem 2024-06-18T02:42:41.020 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=hwmon 2024-06-18T02:42:41.020 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=infiniband 2024-06-18T02:42:41.020 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=ipvs 2024-06-18T02:42:41.020 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=loadavg 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=mdadm 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=meminfo 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=netclass 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=netdev 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=netstat 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=nfs 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=nfsd 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=nvme 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=os 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=pressure 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=rapl 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=schedstat 2024-06-18T02:42:41.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=sockstat 2024-06-18T02:42:41.022 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=softnet 2024-06-18T02:42:41.022 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=stat 2024-06-18T02:42:41.022 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=tapestats 2024-06-18T02:42:41.022 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=textfile 2024-06-18T02:42:41.022 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-06-18T02:42:41.022 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=time 2024-06-18T02:42:41.022 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-06-18T02:42:41.022 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=uname 2024-06-18T02:42:41.022 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=vmstat 2024-06-18T02:42:41.022 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=xfs 2024-06-18T02:42:41.022 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:115 level=info collector=zfs 2024-06-18T02:42:41.022 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.711Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-06-18T02:42:41.022 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[41245]: ts=2024-06-18T02:42:40.712Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-06-18T02:42:41.023 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 bash[41195]: 48a19a2881139dae70078a2e6156c9701734fb1c32859d415a7e3699ea00628f 2024-06-18T02:42:41.023 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:42:40 smithi186 systemd[1]: Started Ceph node-exporter.b for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:42:42.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:41 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:42.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:41 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:42:42.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:41 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:42:42.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:41 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:42:42.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:41 smithi163 ceph-mon[29791]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:42.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:41 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:42.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:41 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:42:42.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:41 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:42:42.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:41 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:42:42.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:41 smithi163 ceph-mon[25094]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:42.267 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:41 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:42.268 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:41 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T02:42:42.268 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:41 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:42:42.268 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:41 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:42:42.268 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:41 smithi186 ceph-mon[25944]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:43.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:43 smithi163 ceph-mon[29791]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:43.518 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:43 smithi163 ceph-mon[25094]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:43.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:43 smithi186 ceph-mon[25944]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:43.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:42:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:42:45.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:44 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:45.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:44 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:44 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:45.774 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:45 smithi163 ceph-mon[29791]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:45.775 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:45 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:45.775 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:45 smithi163 ceph-mon[25094]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:45.775 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:45 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:45 smithi186 ceph-mon[25944]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:45 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:42:47.194 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:47 smithi163 ceph-mon[29791]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:47.195 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:47 smithi163 ceph-mon[25094]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:47.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:47 smithi186 ceph-mon[25944]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:48.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:42:48] "GET /metrics HTTP/1.1" 200 191503 "" "Prometheus/2.33.4" 2024-06-18T02:42:49.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:49 smithi163 ceph-mon[29791]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:49.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:49 smithi163 ceph-mon[25094]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:49.394 INFO:teuthology.orchestra.run.smithi163.stdout:201863462929 2024-06-18T02:42:49.395 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-06-18T02:42:49.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:49 smithi186 ceph-mon[25944]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:50.800 INFO:teuthology.orchestra.run.smithi163.stdout:55834574889 2024-06-18T02:42:50.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-06-18T02:42:51.299 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:51 smithi163 ceph-mon[29791]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:51.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:51 smithi163 ceph-mon[25094]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:51 smithi186 ceph-mon[25944]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:51.980 INFO:teuthology.orchestra.run.smithi163.stdout:167503724566 2024-06-18T02:42:51.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-06-18T02:42:52.798 INFO:teuthology.orchestra.run.smithi163.stdout:111669149728 2024-06-18T02:42:52.799 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-06-18T02:42:53.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:53 smithi163 ceph-mon[29791]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:53.338 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:53 smithi163 ceph-mon[25094]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:53.459 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:53 smithi186 ceph-mon[25944]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:53.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:42:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:42:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:42:54.165 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:54.165 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:55.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:55 smithi163 ceph-mon[25094]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:55.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:55 smithi163 ceph-mon[29791]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:55.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:55 smithi186 ceph-mon[25944]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:56.170 INFO:teuthology.orchestra.run.smithi163.stdout:34359738413 2024-06-18T02:42:56.171 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-06-18T02:42:56.772 INFO:teuthology.orchestra.run.smithi163.stdout:236223201294 2024-06-18T02:42:56.772 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-06-18T02:42:56.779 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:42:57.365 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:57 smithi163 ceph-mon[25094]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:57.365 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:57 smithi163 ceph-mon[29791]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:57.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:57 smithi186 ceph-mon[25944]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:58.050 INFO:teuthology.orchestra.run.smithi163.stdout:55834574891 2024-06-18T02:42:58.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:58 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/233732711' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-06-18T02:42:58.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:58 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/233732711' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-06-18T02:42:58.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:58 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/233732711' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-06-18T02:42:58.758 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:42:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:42:58] "GET /metrics HTTP/1.1" 200 191503 "" "Prometheus/2.33.4" 2024-06-18T02:42:59.502 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:42:59 smithi163 ceph-mon[25094]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:59.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:42:59 smithi163 ceph-mon[29791]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:59.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:42:59 smithi186 ceph-mon[25944]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:42:59.877 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:43:00.424 INFO:teuthology.orchestra.run.smithi163.stdout:133143986204 2024-06-18T02:43:00.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-06-18T02:43:01.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:01 smithi163 ceph-mon[25094]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:01.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:01 smithi163 ceph-mon[29791]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:01.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:01 smithi186 ceph-mon[25944]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:03.270 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:03 smithi163 ceph-mon[29791]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:03.270 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:03 smithi163 ceph-mon[25094]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:03.293 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574889 got 55834574891 for osd.1 2024-06-18T02:43:03.294 DEBUG:teuthology.parallel:result is None 2024-06-18T02:43:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:03 smithi186 ceph-mon[25944]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:03.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:43:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:43:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:43:03.578 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:43:04.808 INFO:teuthology.orchestra.run.smithi163.stdout:77309411366 2024-06-18T02:43:04.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-06-18T02:43:05.418 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:05 smithi163 ceph-mon[29791]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:05.419 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:05 smithi163 ceph-mon[25094]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:05.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:05 smithi186 ceph-mon[25944]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:06.072 INFO:teuthology.orchestra.run.smithi163.stdout:201863462933 2024-06-18T02:43:06.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:06 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1531485363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-06-18T02:43:06.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:06 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1531485363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-06-18T02:43:06.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:06 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1531485363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-06-18T02:43:06.693 INFO:teuthology.orchestra.run.smithi163.stdout:167503724569 2024-06-18T02:43:07.333 INFO:teuthology.orchestra.run.smithi163.stdout:111669149731 2024-06-18T02:43:07.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:07 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1591949996' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-06-18T02:43:07.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:07 smithi163 ceph-mon[29791]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:07.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:07 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1591949996' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-06-18T02:43:07.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:07 smithi163 ceph-mon[25094]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:07.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:07 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1591949996' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-06-18T02:43:07.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:07 smithi186 ceph-mon[25944]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:08.056 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149728 got 111669149731 for osd.3 2024-06-18T02:43:08.056 DEBUG:teuthology.parallel:result is None 2024-06-18T02:43:08.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:08 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1076631145' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-06-18T02:43:08.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:08 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1076631145' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-06-18T02:43:08.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:08 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1076631145' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-06-18T02:43:08.677 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462929 got 201863462933 for osd.6 2024-06-18T02:43:08.678 DEBUG:teuthology.parallel:result is None 2024-06-18T02:43:08.886 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:43:08 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:43:08] "GET /metrics HTTP/1.1" 200 191499 "" "Prometheus/2.33.4" 2024-06-18T02:43:09.206 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724566 got 167503724569 for osd.5 2024-06-18T02:43:09.206 DEBUG:teuthology.parallel:result is None 2024-06-18T02:43:09.429 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:43:09.429 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:43:09.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:09 smithi163 ceph-mon[29791]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:09.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:09 smithi163 ceph-mon[25094]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:09.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:09 smithi186 ceph-mon[25944]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:09.614 INFO:teuthology.orchestra.run.smithi163.stdout:34359738416 2024-06-18T02:43:10.464 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:10 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/978051806' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-06-18T02:43:10.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:10 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/978051806' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-06-18T02:43:10.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:10 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/978051806' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-06-18T02:43:10.825 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738413 got 34359738416 for osd.0 2024-06-18T02:43:10.825 DEBUG:teuthology.parallel:result is None 2024-06-18T02:43:11.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:11 smithi163 ceph-mon[29791]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:11.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:11 smithi163 ceph-mon[25094]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:11.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:11 smithi186 ceph-mon[25944]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:11.602 INFO:teuthology.orchestra.run.smithi163.stdout:236223201297 2024-06-18T02:43:12.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:12 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3953042509' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-06-18T02:43:12.669 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:12 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3953042509' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-06-18T02:43:12.670 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:12 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3953042509' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-06-18T02:43:13.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:13 smithi163 ceph-mon[29791]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:13.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:13 smithi163 ceph-mon[25094]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:13.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:13 smithi186 ceph-mon[25944]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:13.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:43:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:43:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:43:14.164 INFO:teuthology.orchestra.run.smithi163.stdout:133143986208 2024-06-18T02:43:14.536 INFO:teuthology.orchestra.run.smithi163.stdout:77309411369 2024-06-18T02:43:15.001 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986204 got 133143986208 for osd.4 2024-06-18T02:43:15.002 DEBUG:teuthology.parallel:result is None 2024-06-18T02:43:15.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:14 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:15.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:14 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:15.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:14 smithi163 ceph-mon[25094]: Deploying daemon alertmanager.a on smithi163 2024-06-18T02:43:15.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:14 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1893106441' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-06-18T02:43:15.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:14 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/4232234971' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-06-18T02:43:15.283 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:14 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:15.283 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:14 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:15.283 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:14 smithi163 ceph-mon[29791]: Deploying daemon alertmanager.a on smithi163 2024-06-18T02:43:15.283 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:14 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1893106441' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-06-18T02:43:15.283 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:14 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/4232234971' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-06-18T02:43:15.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:14 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:15.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:14 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:15.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:14 smithi186 ceph-mon[25944]: Deploying daemon alertmanager.a on smithi163 2024-06-18T02:43:15.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:14 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1893106441' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-06-18T02:43:15.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:14 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/4232234971' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-06-18T02:43:15.606 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411366 got 77309411369 for osd.2 2024-06-18T02:43:15.606 DEBUG:teuthology.parallel:result is None 2024-06-18T02:43:16.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:15 smithi163 ceph-mon[29791]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:16.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:43:16.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:43:16.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:43:16.188 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:43:16.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:15 smithi163 ceph-mon[25094]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:16.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:43:16.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:43:16.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:43:16.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:43:16.209 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201294 got 236223201297 for osd.7 2024-06-18T02:43:16.209 DEBUG:teuthology.parallel:result is None 2024-06-18T02:43:16.209 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-06-18T02:43:16.210 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph pg dump --format=json 2024-06-18T02:43:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:15 smithi186 ceph-mon[25944]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:43:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:43:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:43:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:43:16.445 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:43:17.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:17 smithi163 ceph-mon[29791]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:17.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:17 smithi163 ceph-mon[25094]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:17.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:17 smithi186 ceph-mon[25944]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:18.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:43:18 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:43:18] "GET /metrics HTTP/1.1" 200 191498 "" "Prometheus/2.33.4" 2024-06-18T02:43:19.098 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:43:19.099 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2024-06-18T02:43:19.427 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:19 smithi163 ceph-mon[25094]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:19.427 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:19 smithi163 ceph-mon[29791]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:19.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:19 smithi186 ceph-mon[25944]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:19.776 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":35,"stamp":"2024-06-18T02:43:19.018117+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48960,"kb_used_data":6144,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912256,"statfs":{"total":767960285184,"available":767910150144,"internally_reserved":0,"allocated":6291456,"data_stored":3510416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001963"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-06-18T02:42:15.091890+0000","last_change":"2024-06-18T02:42:00.515247+0000","last_active":"2024-06-18T02:42:15.091890+0000","last_peered":"2024-06-18T02:42:15.091890+0000","last_clean":"2024-06-18T02:42:15.091890+0000","last_became_active":"2024-06-18T02:42:00.514789+0000","last_became_peered":"2024-06-18T02:42:00.514789+0000","last_unstale":"2024-06-18T02:42:15.091890+0000","last_undegraded":"2024-06-18T02:42:15.091890+0000","last_fullsized":"2024-06-18T02:42:15.091890+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-06-18T02:40:01.063914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-06-18T02:40:01.063914+0000","last_clean_scrub_stamp":"2024-06-18T02:40:01.063914+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-06-19T13:08:18.456964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":6.569,"5min":6.569,"15min":6.569},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.54600000000000004}]},{"osd":1,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.57399999999999995}]},{"osd":2,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.56399999999999995}]},{"osd":3,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.593,"5min":6.593,"15min":6.593},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.53400000000000003}]},{"osd":4,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.39200000000000002}]},{"osd":5,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.443},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.41399999999999998}]},{"osd":6,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.377},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.35399999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462936,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":769000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.59799999999999998}]},{"osd":1,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.433}]},{"osd":2,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.46700000000000003}]},{"osd":3,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.49199999999999999}]},{"osd":4,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.45700000000000002}]},{"osd":5,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.628},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.56499999999999995}]},{"osd":7,"last update":"Tue Jun 18 02:43:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.61499999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:42:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.36599999999999999,"15min":0.34899999999999998},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.434},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.38700000000000001,"15min":0.35299999999999998},"min":{"1min":0.28899999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Tue Jun 18 02:43:06 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.375,"15min":0.36899999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.42299999999999999,"15min":0.40500000000000003},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.39500000000000002}]},{"osd":3,"last update":"Tue Jun 18 02:42:30 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.45800000000000002,"15min":0.46000000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.65500000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.442,"15min":0.443},"min":{"1min":0.33300000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.374}]},{"osd":4,"last update":"Tue Jun 18 02:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.57999999999999996,"15min":0.57199999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.58799999999999997,"15min":0.58499999999999996},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1,"5min":1,"15min":1},"last":0.71499999999999997}]},{"osd":5,"last update":"Tue Jun 18 02:43:15 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.64000000000000001,"15min":0.64600000000000002},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.92500000000000004,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.63800000000000001,"15min":0.63600000000000001},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.92700000000000005,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.74099999999999999}]},{"osd":6,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.68400000000000005}]},{"osd":7,"last update":"Tue Jun 18 02:42:58 2024","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.76600000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738418,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6924,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738228,"statfs":{"total":95995035648,"available":95987945472,"internally_reserved":0,"allocated":1118208,"data_stored":769323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Jun 18 02:42:46 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.34300000000000003,"15min":0.32000000000000001},"min":{"1min":0.27200000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.34799999999999998,"15min":0.32700000000000001},"min":{"1min":0.313,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.35599999999999998}]},{"osd":2,"last update":"Tue Jun 18 02:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39700000000000002,"15min":0.39300000000000002},"min":{"1min":0.27600000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.39600000000000002},"min":{"1min":0.31,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.31}]},{"osd":3,"last update":"Tue Jun 18 02:42:27 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.42399999999999999,"15min":0.41799999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.45100000000000001,"15min":0.44700000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.378}]},{"osd":4,"last update":"Tue Jun 18 02:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.57399999999999995,"15min":0.56599999999999995},"min":{"1min":0.43099999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.57699999999999996,"15min":0.57099999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.60699999999999998}]},{"osd":5,"last update":"Tue Jun 18 02:43:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.624,"15min":0.628},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.74299999999999999}]},{"osd":6,"last update":"Tue Jun 18 02:42:37 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.57799999999999996}]},{"osd":7,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.753},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.61499999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6276,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:43:04 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.35699999999999998},"min":{"1min":0.23000000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.248,"5min":1.248,"15min":1.248},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.38500000000000001,"15min":0.36699999999999999},"min":{"1min":0.29999999999999999,"5min":0.254,"15min":0.254},"max":{"1min":1.1319999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.32500000000000001}]},{"osd":1,"last update":"Tue Jun 18 02:43:04 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.38400000000000001,"15min":0.36399999999999999},"min":{"1min":0.253,"5min":0.251,"15min":0.251},"max":{"1min":1.3029999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40100000000000002,"15min":0.38500000000000001},"min":{"1min":0.24199999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.1639999999999999,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.36899999999999999}]},{"osd":3,"last update":"Tue Jun 18 02:42:25 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.46200000000000002,"15min":0.46500000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.48299999999999998,"15min":0.48499999999999999},"min":{"1min":0.36899999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.40100000000000002}]},{"osd":4,"last update":"Tue Jun 18 02:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.60099999999999998,"15min":0.60299999999999998},"min":{"1min":0.40100000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.534,"5min":1.534,"15min":1.534},"last":0.501},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.59699999999999998,"15min":0.59899999999999998},"min":{"1min":0.442,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":0.53300000000000003}]},{"osd":5,"last update":"Tue Jun 18 02:43:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.57999999999999996,"15min":0.58599999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.628,"5min":0.63700000000000001,"15min":0.63900000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.5009999999999999,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.54500000000000004}]},{"osd":6,"last update":"Tue Jun 18 02:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.52400000000000002}]},{"osd":7,"last update":"Tue Jun 18 02:42:58 2024","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.5609999999999999,"5min":1.5609999999999999,"15min":1.5609999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.619,"5min":1.619,"15min":1.619},"last":0.60699999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149733,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:42:27 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.38300000000000001,"15min":0.379},"min":{"1min":0.28899999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.40600000000000003,"15min":0.39800000000000002},"min":{"1min":0.29699999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.29999999999999999}]},{"osd":1,"last update":"Tue Jun 18 02:42:27 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.432,"15min":0.42799999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.61699999999999999,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.26400000000000001}]},{"osd":2,"last update":"Tue Jun 18 02:42:27 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.46100000000000002,"15min":0.45700000000000002},"min":{"1min":0.309,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.66200000000000003,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.31},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.42999999999999999,"15min":0.42599999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.36099999999999999}]},{"osd":4,"last update":"Tue Jun 18 02:42:49 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.56999999999999995,"15min":0.57199999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.65600000000000003,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.626}]},{"osd":5,"last update":"Tue Jun 18 02:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.55900000000000005,"15min":0.55300000000000005},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.57699999999999996,"15min":0.57099999999999995},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.59999999999999998}]},{"osd":6,"last update":"Tue Jun 18 02:42:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.68700000000000006}]},{"osd":7,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.67000000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986209,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":240130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:42:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53600000000000003,"15min":0.53900000000000003},"min":{"1min":0.33800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.52900000000000003,"15min":0.53100000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.67800000000000005,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.52500000000000002}]},{"osd":1,"last update":"Tue Jun 18 02:42:46 2024","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":0.67100000000000004,"15min":0.58899999999999997},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":15.436,"5min":15.436,"15min":15.436},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.33100000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.68899999999999995,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.51300000000000001}]},{"osd":2,"last update":"Tue Jun 18 02:42:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52100000000000002,"15min":0.52000000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.55900000000000005,"15min":0.56200000000000006},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72699999999999998,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.42699999999999999}]},{"osd":3,"last update":"Tue Jun 18 02:42:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.58099999999999996,"15min":0.58799999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.73399999999999999,"5min":0.745,"15min":0.745},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.57099999999999995,"15min":0.57699999999999996},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.57199999999999995}]},{"osd":5,"last update":"Tue Jun 18 02:43:11 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.41499999999999998,"15min":0.40799999999999997},"min":{"1min":0.29499999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.44800000000000001,"15min":0.442},"min":{"1min":0.33100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.47899999999999998}]},{"osd":6,"last update":"Tue Jun 18 02:42:35 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.498},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.60499999999999998}]},{"osd":7,"last update":"Tue Jun 18 02:42:56 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.59499999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724572,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.53000000000000003,"15min":0.53300000000000003},"min":{"1min":0.35999999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.68400000000000005,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.51000000000000001,"5min":1.2130000000000001,"15min":1.3300000000000001},"min":{"1min":0.39300000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.63200000000000001,"5min":19.501999999999999,"15min":19.501999999999999},"last":0.51200000000000001}]},{"osd":1,"last update":"Tue Jun 18 02:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.54300000000000004,"15min":0.54900000000000004},"min":{"1min":0.35599999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.69799999999999995,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.499},{"interface":"front","average":{"1min":0.495,"5min":1.1950000000000001,"15min":1.3120000000000001},"min":{"1min":0.35199999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67000000000000004,"5min":19.463999999999999,"15min":19.463999999999999},"last":0.45400000000000001}]},{"osd":2,"last update":"Tue Jun 18 02:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54200000000000004,"15min":0.54700000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.63600000000000001,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.51400000000000001,"5min":1.2190000000000001,"15min":1.337},"min":{"1min":0.38,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.64900000000000002,"5min":19.452999999999999,"15min":19.452999999999999},"last":0.32300000000000001}]},{"osd":3,"last update":"Tue Jun 18 02:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.53600000000000003,"15min":0.53900000000000003},"min":{"1min":0.39000000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.67400000000000004,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.437},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.56000000000000005,"15min":0.56799999999999995},"min":{"1min":0.372,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.65400000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.53400000000000003}]},{"osd":4,"last update":"Tue Jun 18 02:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.40899999999999997,"15min":0.40500000000000003},"min":{"1min":0.28699999999999998,"5min":0.248,"15min":0.248},"max":{"1min":0.65800000000000003,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.42399999999999999,"15min":0.41999999999999998},"min":{"1min":0.32700000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.64200000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.54900000000000004}]},{"osd":6,"last update":"Tue Jun 18 02:42:34 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.56000000000000005}]},{"osd":7,"last update":"Tue Jun 18 02:43:01 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.57099999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-06-18T02:43:19.777 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph pg dump --format=json 2024-06-18T02:43:20.323 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:20 smithi163 ceph-mon[29791]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-18T02:43:20.324 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:20 smithi163 ceph-mon[25094]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-18T02:43:20.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:20 smithi186 ceph-mon[25944]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-18T02:43:20.651 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:43:21.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:21 smithi163 ceph-mon[29791]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:21 smithi163 ceph-mon[25094]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:21.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:21 smithi186 ceph-mon[25944]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:23.161 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:23 smithi163 ceph-mon[29791]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:23.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:23 smithi163 ceph-mon[25094]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:23.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:23 smithi186 ceph-mon[25944]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:23.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:43:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:43:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:43:23.574 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:43:23.575 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2024-06-18T02:43:24.202 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:24 smithi163 ceph-mon[29791]: from='client.24520 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-18T02:43:24.202 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:24 smithi163 ceph-mon[25094]: from='client.24520 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-18T02:43:24.203 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:24 smithi163 systemd[1]: Starting Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:43:24.526 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-06-18T02:43:23.018742+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48960,"kb_used_data":6144,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912256,"statfs":{"total":767960285184,"available":767910150144,"internally_reserved":0,"allocated":6291456,"data_stored":3510416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001912"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-06-18T02:42:15.091890+0000","last_change":"2024-06-18T02:42:00.515247+0000","last_active":"2024-06-18T02:42:15.091890+0000","last_peered":"2024-06-18T02:42:15.091890+0000","last_clean":"2024-06-18T02:42:15.091890+0000","last_became_active":"2024-06-18T02:42:00.514789+0000","last_became_peered":"2024-06-18T02:42:00.514789+0000","last_unstale":"2024-06-18T02:42:15.091890+0000","last_undegraded":"2024-06-18T02:42:15.091890+0000","last_fullsized":"2024-06-18T02:42:15.091890+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-06-18T02:40:01.063914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-06-18T02:40:01.063914+0000","last_clean_scrub_stamp":"2024-06-18T02:40:01.063914+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-06-19T13:08:18.456964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201300,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":6.569,"5min":6.569,"15min":6.569},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.80400000000000005}]},{"osd":1,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.70899999999999996}]},{"osd":2,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.68799999999999994}]},{"osd":3,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.593,"5min":6.593,"15min":6.593},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.745}]},{"osd":4,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.63400000000000001}]},{"osd":5,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.752}]},{"osd":6,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.69599999999999995}]}]},{"osd":6,"up_from":47,"seq":201863462937,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":769000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.59799999999999998}]},{"osd":1,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.433}]},{"osd":2,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.46700000000000003}]},{"osd":3,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.49199999999999999}]},{"osd":4,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.45700000000000002}]},{"osd":5,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.628},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.56499999999999995}]},{"osd":7,"last update":"Tue Jun 18 02:43:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.61499999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574896,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:42:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.36599999999999999,"15min":0.34899999999999998},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.38700000000000001,"15min":0.35299999999999998},"min":{"1min":0.28899999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.32200000000000001}]},{"osd":2,"last update":"Tue Jun 18 02:43:06 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.375,"15min":0.36899999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.252},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.42299999999999999,"15min":0.40500000000000003},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.35099999999999998}]},{"osd":3,"last update":"Tue Jun 18 02:42:30 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.45800000000000002,"15min":0.46000000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.65500000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.436,"5min":0.442,"15min":0.443},"min":{"1min":0.33300000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.35999999999999999}]},{"osd":4,"last update":"Tue Jun 18 02:42:50 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.57999999999999996,"15min":0.57199999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.58799999999999997,"15min":0.58499999999999996},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1,"5min":1,"15min":1},"last":0.64000000000000001}]},{"osd":5,"last update":"Tue Jun 18 02:43:15 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.64000000000000001,"15min":0.64600000000000002},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.92500000000000004,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.63800000000000001,"15min":0.63600000000000001},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.92700000000000005,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.69599999999999995}]},{"osd":6,"last update":"Tue Jun 18 02:42:32 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.56699999999999995}]},{"osd":7,"last update":"Tue Jun 18 02:42:58 2024","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.70999999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6924,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738228,"statfs":{"total":95995035648,"available":95987945472,"internally_reserved":0,"allocated":1118208,"data_stored":769323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Jun 18 02:42:46 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.34300000000000003,"15min":0.32000000000000001},"min":{"1min":0.27200000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.34799999999999998,"15min":0.32700000000000001},"min":{"1min":0.313,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.45400000000000001}]},{"osd":2,"last update":"Tue Jun 18 02:43:07 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39700000000000002,"15min":0.39300000000000002},"min":{"1min":0.27600000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.39600000000000002},"min":{"1min":0.31,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.40600000000000003}]},{"osd":3,"last update":"Tue Jun 18 02:42:27 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.42399999999999999,"15min":0.41799999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.45100000000000001,"15min":0.44700000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.51500000000000001}]},{"osd":4,"last update":"Tue Jun 18 02:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.57399999999999995,"15min":0.56599999999999995},"min":{"1min":0.43099999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.57699999999999996,"15min":0.57099999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.628}]},{"osd":5,"last update":"Tue Jun 18 02:43:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.624,"15min":0.628},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.78900000000000003}]},{"osd":6,"last update":"Tue Jun 18 02:42:37 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.746},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.67100000000000004}]},{"osd":7,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.89100000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6276,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:43:04 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.35699999999999998},"min":{"1min":0.23000000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.248,"5min":1.248,"15min":1.248},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.38500000000000001,"15min":0.36699999999999999},"min":{"1min":0.29999999999999999,"5min":0.254,"15min":0.254},"max":{"1min":1.1319999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.35299999999999998}]},{"osd":1,"last update":"Tue Jun 18 02:43:04 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.38400000000000001,"15min":0.36399999999999999},"min":{"1min":0.253,"5min":0.251,"15min":0.251},"max":{"1min":1.3029999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40100000000000002,"15min":0.38500000000000001},"min":{"1min":0.24199999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.1639999999999999,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.33900000000000002}]},{"osd":3,"last update":"Tue Jun 18 02:42:25 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.46200000000000002,"15min":0.46500000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.48299999999999998,"15min":0.48499999999999999},"min":{"1min":0.36899999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.30199999999999999}]},{"osd":4,"last update":"Tue Jun 18 02:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.60099999999999998,"15min":0.60299999999999998},"min":{"1min":0.40100000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.534,"5min":1.534,"15min":1.534},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.59699999999999998,"15min":0.59899999999999998},"min":{"1min":0.442,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":0.67600000000000005}]},{"osd":5,"last update":"Tue Jun 18 02:43:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.57999999999999996,"15min":0.58599999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.628,"5min":0.63700000000000001,"15min":0.63900000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.5009999999999999,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.56100000000000005}]},{"osd":6,"last update":"Tue Jun 18 02:42:33 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.63100000000000001}]},{"osd":7,"last update":"Tue Jun 18 02:42:58 2024","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.5609999999999999,"5min":1.5609999999999999,"15min":1.5609999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.619,"5min":1.619,"15min":1.619},"last":0.60699999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:42:27 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.38300000000000001,"15min":0.379},"min":{"1min":0.28899999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.40600000000000003,"15min":0.39800000000000002},"min":{"1min":0.29699999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.374}]},{"osd":1,"last update":"Tue Jun 18 02:42:27 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.432,"15min":0.42799999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.61699999999999999,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.32700000000000001}]},{"osd":2,"last update":"Tue Jun 18 02:42:27 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.46100000000000002,"15min":0.45700000000000002},"min":{"1min":0.309,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.66200000000000003,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.42999999999999999,"15min":0.42599999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.35399999999999998}]},{"osd":4,"last update":"Tue Jun 18 02:42:49 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.56999999999999995,"15min":0.57199999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.65600000000000003,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.64800000000000002}]},{"osd":5,"last update":"Tue Jun 18 02:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.55900000000000005,"15min":0.55300000000000005},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.57699999999999996,"15min":0.57099999999999995},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.62}]},{"osd":6,"last update":"Tue Jun 18 02:42:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.63400000000000001}]},{"osd":7,"last update":"Tue Jun 18 02:42:57 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.60299999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986210,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":240130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:42:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53600000000000003,"15min":0.53900000000000003},"min":{"1min":0.33800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.52900000000000003,"15min":0.53100000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.67800000000000005,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.52500000000000002}]},{"osd":1,"last update":"Tue Jun 18 02:42:46 2024","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":0.67100000000000004,"15min":0.58899999999999997},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":15.436,"5min":15.436,"15min":15.436},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.33100000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.68899999999999995,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.51300000000000001}]},{"osd":2,"last update":"Tue Jun 18 02:42:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52100000000000002,"15min":0.52000000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.55900000000000005,"15min":0.56200000000000006},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72699999999999998,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.42699999999999999}]},{"osd":3,"last update":"Tue Jun 18 02:42:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.58099999999999996,"15min":0.58799999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.73399999999999999,"5min":0.745,"15min":0.745},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.57099999999999995,"15min":0.57699999999999996},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.57199999999999995}]},{"osd":5,"last update":"Tue Jun 18 02:43:11 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.41499999999999998,"15min":0.40799999999999997},"min":{"1min":0.29499999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.44800000000000001,"15min":0.442},"min":{"1min":0.33100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.47899999999999998}]},{"osd":6,"last update":"Tue Jun 18 02:42:35 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.498},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.60499999999999998}]},{"osd":7,"last update":"Tue Jun 18 02:42:56 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.59499999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724573,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 18 02:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.53000000000000003,"15min":0.53300000000000003},"min":{"1min":0.35999999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.68400000000000005,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.51000000000000001,"5min":1.2130000000000001,"15min":1.3300000000000001},"min":{"1min":0.39300000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.63200000000000001,"5min":19.501999999999999,"15min":19.501999999999999},"last":0.67900000000000005}]},{"osd":1,"last update":"Tue Jun 18 02:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.54300000000000004,"15min":0.54900000000000004},"min":{"1min":0.35599999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.69799999999999995,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.502},{"interface":"front","average":{"1min":0.495,"5min":1.1950000000000001,"15min":1.3120000000000001},"min":{"1min":0.35199999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67000000000000004,"5min":19.463999999999999,"15min":19.463999999999999},"last":0.51600000000000001}]},{"osd":2,"last update":"Tue Jun 18 02:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54200000000000004,"15min":0.54700000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.63600000000000001,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.51400000000000001,"5min":1.2190000000000001,"15min":1.337},"min":{"1min":0.38,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.64900000000000002,"5min":19.452999999999999,"15min":19.452999999999999},"last":0.623}]},{"osd":3,"last update":"Tue Jun 18 02:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.53600000000000003,"15min":0.53900000000000003},"min":{"1min":0.39000000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.67400000000000004,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.56000000000000005,"15min":0.56799999999999995},"min":{"1min":0.372,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.65400000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.70999999999999996}]},{"osd":4,"last update":"Tue Jun 18 02:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.40899999999999997,"15min":0.40500000000000003},"min":{"1min":0.28699999999999998,"5min":0.248,"15min":0.248},"max":{"1min":0.65800000000000003,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.42399999999999999,"15min":0.41999999999999998},"min":{"1min":0.32700000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.64200000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.61799999999999999}]},{"osd":6,"last update":"Tue Jun 18 02:42:34 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.75900000000000001}]},{"osd":7,"last update":"Tue Jun 18 02:43:01 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.79900000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-06-18T02:43:24.527 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-06-18T02:43:24.527 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-06-18T02:43:24.528 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-06-18T02:43:24.528 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- ceph health --format=json 2024-06-18T02:43:24.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:24 smithi186 ceph-mon[25944]: from='client.24520 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-18T02:43:25.045 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:24 smithi163 podman[48111]: 2024-06-18 02:43:24.669330149 +0000 UTC m=+0.017654895 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-06-18T02:43:25.046 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:24 smithi163 podman[48111]: 2024-06-18 02:43:24.804550135 +0000 UTC m=+0.152874879 volume create aea8072f16c068e6439d08377c373dc9ccbbecb8ae91465baeb06a1438b4cf13 2024-06-18T02:43:25.046 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:24 smithi163 podman[48111]: 2024-06-18 02:43:24.888254333 +0000 UTC m=+0.236579126 container create b13a058d80866bbd98ee493e4db0af7e0fcd0f1d1de84b565b7d36fc1a76ab98 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:25.389 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/mon.a/config 2024-06-18T02:43:25.397 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:25 smithi186 ceph-mon[25944]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:25.512 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:25 smithi163 ceph-mon[29791]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:25.512 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:25 smithi163 ceph-mon[25094]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:25.513 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:25 smithi163 podman[48111]: 2024-06-18 02:43:25.174452067 +0000 UTC m=+0.522776814 container init b13a058d80866bbd98ee493e4db0af7e0fcd0f1d1de84b565b7d36fc1a76ab98 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:25.513 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:25 smithi163 podman[48111]: 2024-06-18 02:43:25.177345313 +0000 UTC m=+0.525670052 container start b13a058d80866bbd98ee493e4db0af7e0fcd0f1d1de84b565b7d36fc1a76ab98 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:25.513 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[48138]: level=info ts=2024-06-18T02:43:25.187Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-06-18T02:43:25.513 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[48138]: level=info ts=2024-06-18T02:43:25.187Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-06-18T02:43:25.513 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[48138]: level=info ts=2024-06-18T02:43:25.188Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.163 port=9094 2024-06-18T02:43:25.513 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[48138]: level=info ts=2024-06-18T02:43:25.190Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-06-18T02:43:25.513 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[48138]: level=info ts=2024-06-18T02:43:25.214Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-18T02:43:25.513 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[48138]: level=info ts=2024-06-18T02:43:25.215Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-18T02:43:25.513 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[48138]: level=info ts=2024-06-18T02:43:25.217Z caller=main.go:518 msg=Listening address=:9093 2024-06-18T02:43:25.513 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[48138]: level=info ts=2024-06-18T02:43:25.217Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-06-18T02:43:25.513 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:25 smithi163 bash[48111]: b13a058d80866bbd98ee493e4db0af7e0fcd0f1d1de84b565b7d36fc1a76ab98 2024-06-18T02:43:25.513 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:25 smithi163 systemd[1]: Started Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:43:26.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:26.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:26.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:26.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-18T02:43:26.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-18T02:43:26.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:26.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[29791]: Deploying daemon grafana.a on smithi186 2024-06-18T02:43:26.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:26.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:26.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:26.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-18T02:43:26.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-18T02:43:26.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:26.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:26 smithi163 ceph-mon[25094]: Deploying daemon grafana.a on smithi186 2024-06-18T02:43:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:26 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:26 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:26 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:26 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-18T02:43:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:26 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-18T02:43:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:26 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:26 smithi186 ceph-mon[25944]: Deploying daemon grafana.a on smithi186 2024-06-18T02:43:27.096 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:43:27.096 INFO:teuthology.orchestra.run.smithi163.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-06-18T02:43:27.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:27 smithi163 ceph-mon[29791]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:27.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:27 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/931418228' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-06-18T02:43:27.447 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[48138]: level=info ts=2024-06-18T02:43:27.191Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000798556s 2024-06-18T02:43:27.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:27 smithi163 ceph-mon[25094]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:27.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:27 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/931418228' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-06-18T02:43:27.730 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-06-18T02:43:27.731 INFO:tasks.cephadm:Setup complete, yielding 2024-06-18T02:43:27.731 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-18T02:43:27.743 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi163.front.sepia.ceph.com 2024-06-18T02:43:27.743 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest false --force' 2024-06-18T02:43:27.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:27 smithi186 ceph-mon[25944]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:27 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/931418228' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-06-18T02:43:29.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:43:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:43:28] "GET /metrics HTTP/1.1" 200 191498 "" "Prometheus/2.33.4" 2024-06-18T02:43:29.326 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:29 smithi163 ceph-mon[29791]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:29.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:29 smithi163 ceph-mon[25094]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:29.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:29 smithi186 ceph-mon[25944]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:30.284 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-18T02:43:30.293 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi163.front.sepia.ceph.com 2024-06-18T02:43:30.293 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-06-18T02:43:30.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:30 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' 2024-06-18T02:43:30.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:30 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:30.857 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:30 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' 2024-06-18T02:43:30.857 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:30 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:31.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:30 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' 2024-06-18T02:43:31.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:30 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:32.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:31 smithi163 ceph-mon[29791]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:32.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:31 smithi163 ceph-mon[25094]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:32.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:31 smithi186 ceph-mon[25944]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:33.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:43:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:43:33.767 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T02:43:33.768 INFO:teuthology.orchestra.run.smithi163.stdout: "id": "3e8b7a8b-90be-43d6-93da-cdb514a88b8d", 2024-06-18T02:43:33.768 INFO:teuthology.orchestra.run.smithi163.stdout: "name": "r", 2024-06-18T02:43:33.768 INFO:teuthology.orchestra.run.smithi163.stdout: "current_period": "42b7a9ac-6cda-42af-a912-214a7b109b03", 2024-06-18T02:43:33.768 INFO:teuthology.orchestra.run.smithi163.stdout: "epoch": 1 2024-06-18T02:43:33.768 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T02:43:34.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:33 smithi163 ceph-mon[25094]: osdmap e61: 8 total, 8 up, 8 in 2024-06-18T02:43:34.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:33 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2935924068' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-06-18T02:43:34.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:33 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-06-18T02:43:34.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:33 smithi163 ceph-mon[25094]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:34.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:33 smithi163 ceph-mon[29791]: osdmap e61: 8 total, 8 up, 8 in 2024-06-18T02:43:34.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:33 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2935924068' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-06-18T02:43:34.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:33 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-06-18T02:43:34.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:33 smithi163 ceph-mon[29791]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:33 smithi186 ceph-mon[25944]: osdmap e61: 8 total, 8 up, 8 in 2024-06-18T02:43:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:33 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2935924068' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-06-18T02:43:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:33 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-06-18T02:43:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:33 smithi186 ceph-mon[25944]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:34.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-06-18T02:43:34.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:34 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-06-18T02:43:34.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:34 smithi186 ceph-mon[25944]: osdmap e62: 8 total, 8 up, 8 in 2024-06-18T02:43:35.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:34 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-06-18T02:43:35.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:34 smithi163 ceph-mon[29791]: osdmap e62: 8 total, 8 up, 8 in 2024-06-18T02:43:35.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:34 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-06-18T02:43:35.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:34 smithi163 ceph-mon[25094]: osdmap e62: 8 total, 8 up, 8 in 2024-06-18T02:43:35.292 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:35 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[48138]: level=info ts=2024-06-18T02:43:35.193Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003512349s 2024-06-18T02:43:35.986 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:35 smithi163 ceph-mon[25094]: osdmap e63: 8 total, 8 up, 8 in 2024-06-18T02:43:35.986 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:35 smithi163 ceph-mon[25094]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:35.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:35 smithi163 ceph-mon[29791]: osdmap e63: 8 total, 8 up, 8 in 2024-06-18T02:43:35.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:35 smithi163 ceph-mon[29791]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:36.000 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:35 smithi186 ceph-mon[25944]: osdmap e63: 8 total, 8 up, 8 in 2024-06-18T02:43:36.000 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:35 smithi186 ceph-mon[25944]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:36.061 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T02:43:36.062 INFO:teuthology.orchestra.run.smithi163.stdout: "id": "8ba594d6-06fa-4b90-a7de-629262a6b495", 2024-06-18T02:43:36.062 INFO:teuthology.orchestra.run.smithi163.stdout: "name": "default", 2024-06-18T02:43:36.062 INFO:teuthology.orchestra.run.smithi163.stdout: "api_name": "default", 2024-06-18T02:43:36.062 INFO:teuthology.orchestra.run.smithi163.stdout: "is_master": "true", 2024-06-18T02:43:36.062 INFO:teuthology.orchestra.run.smithi163.stdout: "endpoints": [], 2024-06-18T02:43:36.062 INFO:teuthology.orchestra.run.smithi163.stdout: "hostnames": [], 2024-06-18T02:43:36.062 INFO:teuthology.orchestra.run.smithi163.stdout: "hostnames_s3website": [], 2024-06-18T02:43:36.062 INFO:teuthology.orchestra.run.smithi163.stdout: "master_zone": "", 2024-06-18T02:43:36.062 INFO:teuthology.orchestra.run.smithi163.stdout: "zones": [], 2024-06-18T02:43:36.062 INFO:teuthology.orchestra.run.smithi163.stdout: "placement_targets": [], 2024-06-18T02:43:36.062 INFO:teuthology.orchestra.run.smithi163.stdout: "default_placement": "", 2024-06-18T02:43:36.062 INFO:teuthology.orchestra.run.smithi163.stdout: "realm_id": "3e8b7a8b-90be-43d6-93da-cdb514a88b8d", 2024-06-18T02:43:36.062 INFO:teuthology.orchestra.run.smithi163.stdout: "sync_policy": { 2024-06-18T02:43:36.063 INFO:teuthology.orchestra.run.smithi163.stdout: "groups": [] 2024-06-18T02:43:36.063 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T02:43:36.063 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T02:43:36.657 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-06-18T02:43:37.227 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:37 smithi163 ceph-mon[29791]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-06-18T02:43:37.227 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:37 smithi163 ceph-mon[25094]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-06-18T02:43:37.397 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:37 smithi186 ceph-mon[25944]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-06-18T02:43:38.390 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T02:43:38.391 INFO:teuthology.orchestra.run.smithi163.stdout: "id": "a82b479e-2b6b-407c-b5ce-5177fe7a57d3", 2024-06-18T02:43:38.391 INFO:teuthology.orchestra.run.smithi163.stdout: "name": "z", 2024-06-18T02:43:38.391 INFO:teuthology.orchestra.run.smithi163.stdout: "domain_root": "z.rgw.meta:root", 2024-06-18T02:43:38.391 INFO:teuthology.orchestra.run.smithi163.stdout: "control_pool": "z.rgw.control", 2024-06-18T02:43:38.391 INFO:teuthology.orchestra.run.smithi163.stdout: "gc_pool": "z.rgw.log:gc", 2024-06-18T02:43:38.391 INFO:teuthology.orchestra.run.smithi163.stdout: "lc_pool": "z.rgw.log:lc", 2024-06-18T02:43:38.391 INFO:teuthology.orchestra.run.smithi163.stdout: "log_pool": "z.rgw.log", 2024-06-18T02:43:38.391 INFO:teuthology.orchestra.run.smithi163.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-06-18T02:43:38.391 INFO:teuthology.orchestra.run.smithi163.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-06-18T02:43:38.391 INFO:teuthology.orchestra.run.smithi163.stdout: "roles_pool": "z.rgw.meta:roles", 2024-06-18T02:43:38.391 INFO:teuthology.orchestra.run.smithi163.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-06-18T02:43:38.391 INFO:teuthology.orchestra.run.smithi163.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-06-18T02:43:38.391 INFO:teuthology.orchestra.run.smithi163.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: "otp_pool": "z.rgw.otp", 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: "system_key": { 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: "access_key": "", 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: "secret_key": "" 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: "placement_pools": [ 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: { 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: "key": "default-placement", 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: "val": { 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: "index_pool": "z.rgw.buckets.index", 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: "storage_classes": { 2024-06-18T02:43:38.392 INFO:teuthology.orchestra.run.smithi163.stdout: "STANDARD": { 2024-06-18T02:43:38.393 INFO:teuthology.orchestra.run.smithi163.stdout: "data_pool": "z.rgw.buckets.data" 2024-06-18T02:43:38.393 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T02:43:38.393 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:43:38.393 INFO:teuthology.orchestra.run.smithi163.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-06-18T02:43:38.393 INFO:teuthology.orchestra.run.smithi163.stdout: "index_type": 0 2024-06-18T02:43:38.393 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T02:43:38.393 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T02:43:38.393 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2024-06-18T02:43:38.393 INFO:teuthology.orchestra.run.smithi163.stdout: "realm_id": "3e8b7a8b-90be-43d6-93da-cdb514a88b8d", 2024-06-18T02:43:38.393 INFO:teuthology.orchestra.run.smithi163.stdout: "notif_pool": "z.rgw.log:notif" 2024-06-18T02:43:38.393 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T02:43:38.499 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 podman[41928]: 2024-06-18 02:43:38.354267572 +0000 UTC m=+0.017421069 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-06-18T02:43:38.499 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 podman[41928]: 2024-06-18 02:43:38.499610227 +0000 UTC m=+0.162763696 container create f134c5a8fb6d01a86deb8a35b23d279f2a7c5a8d79a1589fcee979c20615ef51 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, maintainer=Paul Cuzner , version=8.5, release=236.1648460182, io.openshift.expose-services=, architecture=x86_64, io.buildah.version=1.24.2, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-03-28T10:36:18.413762) 2024-06-18T02:43:38.750 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 podman[41928]: 2024-06-18 02:43:38.701802 +0000 UTC m=+0.364955488 container init f134c5a8fb6d01a86deb8a35b23d279f2a7c5a8d79a1589fcee979c20615ef51 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-03-28T10:36:18.413762, name=ubi8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., release=236.1648460182, io.openshift.tags=base rhel8) 2024-06-18T02:43:38.750 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 podman[41928]: 2024-06-18 02:43:38.705016414 +0000 UTC m=+0.368169886 container start f134c5a8fb6d01a86deb8a35b23d279f2a7c5a8d79a1589fcee979c20615ef51 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, maintainer=Paul Cuzner , vendor=Red Hat, Inc., release=236.1648460182, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, io.openshift.expose-services=, architecture=x86_64, io.openshift.tags=base rhel8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, com.redhat.component=ubi8-container, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2) 2024-06-18T02:43:38.750 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-06-18T02:43:38.750 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-06-18T02:43:38.750 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-06-18T02:43:38.750 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-06-18T02:43:38.750 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-06-18T02:43:38.751 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-06-18T02:43:38.751 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-06-18T02:43:38.751 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-06-18T02:43:38.751 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-06-18T02:43:38.751 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-06-18T02:43:38.751 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-06-18T02:43:38.751 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-06-18T02:43:38.751 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-06-18T02:43:38.751 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="App mode production" logger=settings 2024-06-18T02:43:38.753 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-06-18T02:43:38.753 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-06-18T02:43:38.753 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-06-18T02:43:38.753 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-06-18T02:43:38.753 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-06-18T02:43:38.753 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-06-18T02:43:38.753 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-06-18T02:43:38.753 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-06-18T02:43:38.753 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-06-18T02:43:38.753 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-06-18T02:43:38.754 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-06-18T02:43:38.754 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-06-18T02:43:38.754 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-06-18T02:43:38.754 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-06-18T02:43:38.754 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-06-18T02:43:38.754 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-06-18T02:43:38.754 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-06-18T02:43:38.754 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-06-18T02:43:38.893 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:43:38 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:43:38] "GET /metrics HTTP/1.1" 200 191491 "" "Prometheus/2.33.4" 2024-06-18T02:43:39.002 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-06-18T02:43:39.003 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-06-18T02:43:39.003 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-06-18T02:43:39.003 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-06-18T02:43:39.003 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-06-18T02:43:39.003 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-06-18T02:43:39.003 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-06-18T02:43:39.003 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-06-18T02:43:39.003 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-06-18T02:43:39.003 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-06-18T02:43:39.004 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-06-18T02:43:39.004 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-06-18T02:43:39.004 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-06-18T02:43:39.004 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-06-18T02:43:39.004 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-06-18T02:43:39.004 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-06-18T02:43:39.004 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-06-18T02:43:39.004 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-06-18T02:43:39.004 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-06-18T02:43:39.004 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-06-18T02:43:39.004 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 bash[41928]: f134c5a8fb6d01a86deb8a35b23d279f2a7c5a8d79a1589fcee979c20615ef51 2024-06-18T02:43:39.005 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-06-18T02:43:39.005 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 systemd[1]: Started Ceph grafana.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:43:39.005 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-06-18T02:43:39.005 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-06-18T02:43:39.005 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-06-18T02:43:39.005 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-06-18T02:43:39.005 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-06-18T02:43:39.005 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-06-18T02:43:39.005 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-06-18T02:43:39.005 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-06-18T02:43:39.005 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-06-18T02:43:39.005 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-06-18T02:43:39.006 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-06-18T02:43:39.006 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-06-18T02:43:39.006 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-06-18T02:43:39.006 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-06-18T02:43:39.006 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-06-18T02:43:39.006 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-06-18T02:43:39.006 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-06-18T02:43:39.006 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-06-18T02:43:39.006 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-06-18T02:43:39.006 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-06-18T02:43:39.006 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-06-18T02:43:39.006 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-06-18T02:43:39.006 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-06-18T02:43:39.007 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-06-18T02:43:39.007 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-06-18T02:43:39.007 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-06-18T02:43:39.007 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-06-18T02:43:39.007 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-06-18T02:43:39.007 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-06-18T02:43:39.007 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-06-18T02:43:39.007 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-06-18T02:43:39.007 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-06-18T02:43:39.007 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-06-18T02:43:39.007 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-06-18T02:43:39.007 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-06-18T02:43:39.007 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-06-18T02:43:39.008 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-06-18T02:43:39.008 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-06-18T02:43:39.008 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-06-18T02:43:39.009 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-06-18T02:43:39.009 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-06-18T02:43:39.009 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-06-18T02:43:39.009 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-06-18T02:43:39.009 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-06-18T02:43:39.009 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-06-18T02:43:39.009 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-06-18T02:43:39.009 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-06-18T02:43:39.010 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-06-18T02:43:39.010 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-06-18T02:43:39.010 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-06-18T02:43:39.010 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-06-18T02:43:39.010 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-06-18T02:43:39.010 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-06-18T02:43:39.010 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-06-18T02:43:39.010 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-06-18T02:43:39.010 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-06-18T02:43:39.010 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-06-18T02:43:39.010 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-06-18T02:43:39.010 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-06-18T02:43:39.011 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-06-18T02:43:39.011 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-06-18T02:43:39.011 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-06-18T02:43:39.011 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-06-18T02:43:39.011 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-06-18T02:43:39.011 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-06-18T02:43:39.011 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-06-18T02:43:39.011 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-06-18T02:43:39.011 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-06-18T02:43:39.011 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-06-18T02:43:39.011 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-06-18T02:43:39.011 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-06-18T02:43:39.011 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-06-18T02:43:39.012 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-06-18T02:43:39.012 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-06-18T02:43:39.012 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-06-18T02:43:39.012 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-06-18T02:43:39.012 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-06-18T02:43:39.012 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-06-18T02:43:39.012 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-06-18T02:43:39.012 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-06-18T02:43:39.012 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-06-18T02:43:39.012 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-06-18T02:43:39.012 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-06-18T02:43:39.012 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-06-18T02:43:39.012 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-06-18T02:43:39.013 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-06-18T02:43:39.014 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-06-18T02:43:39.015 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-06-18T02:43:39.015 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:38+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-06-18T02:43:39.015 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-06-18T02:43:39.015 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-06-18T02:43:39.015 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-06-18T02:43:39.015 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-06-18T02:43:39.015 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-06-18T02:43:39.255 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-06-18T02:43:39.255 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-06-18T02:43:39.255 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-06-18T02:43:39.255 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-06-18T02:43:39.255 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-06-18T02:43:39.256 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-06-18T02:43:39.256 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-06-18T02:43:39.256 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-06-18T02:43:39.256 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-06-18T02:43:39.256 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-06-18T02:43:39.256 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-06-18T02:43:39.256 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-06-18T02:43:39.256 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-06-18T02:43:39.256 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-06-18T02:43:39.256 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-06-18T02:43:39.256 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-06-18T02:43:39.257 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-06-18T02:43:39.257 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-06-18T02:43:39.257 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-06-18T02:43:39.257 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-06-18T02:43:39.257 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-06-18T02:43:39.257 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-06-18T02:43:39.257 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-06-18T02:43:39.257 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-06-18T02:43:39.257 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-06-18T02:43:39.257 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-06-18T02:43:39.257 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-06-18T02:43:39.257 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-06-18T02:43:39.258 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-06-18T02:43:39.258 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-06-18T02:43:39.258 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-06-18T02:43:39.258 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-06-18T02:43:39.258 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-06-18T02:43:39.258 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-06-18T02:43:39.258 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-06-18T02:43:39.258 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-06-18T02:43:39.258 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-06-18T02:43:39.258 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-06-18T02:43:39.258 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-06-18T02:43:39.259 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-06-18T02:43:39.259 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-06-18T02:43:39.259 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-06-18T02:43:39.260 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-06-18T02:43:39.261 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-06-18T02:43:39.262 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-06-18T02:43:39.262 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-06-18T02:43:39.262 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-06-18T02:43:39.262 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-06-18T02:43:39.262 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-06-18T02:43:39.262 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-06-18T02:43:39.262 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-06-18T02:43:39.262 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-06-18T02:43:39.262 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-06-18T02:43:39.262 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-06-18T02:43:39.262 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-06-18T02:43:39.262 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-06-18T02:43:39.262 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-06-18T02:43:39.263 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-06-18T02:43:39.264 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-06-18T02:43:39.265 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-06-18T02:43:39.266 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-06-18T02:43:39.267 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-06-18T02:43:39.268 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-06-18T02:43:39.268 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-06-18T02:43:39.268 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-06-18T02:43:39.268 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-06-18T02:43:39.268 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-06-18T02:43:39.268 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-06-18T02:43:39.268 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-06-18T02:43:39.268 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-06-18T02:43:39.268 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-06-18T02:43:39.268 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-06-18T02:43:39.268 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-06-18T02:43:39.268 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-06-18T02:43:39.268 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-06-18T02:43:39.569 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-06-18T02:43:39.569 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-06-18T02:43:39.569 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-06-18T02:43:39.569 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-06-18T02:43:39.569 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-06-18T02:43:39.569 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-06-18T02:43:39.569 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-06-18T02:43:39.569 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-06-18T02:43:39.569 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-06-18T02:43:39.569 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-06-18T02:43:39.569 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-06-18T02:43:39.570 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-06-18T02:43:39.570 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-06-18T02:43:39.570 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-06-18T02:43:39.570 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-06-18T02:43:39.570 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-06-18T02:43:39.570 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-06-18T02:43:39.570 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-06-18T02:43:39.573 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-06-18T02:43:39.573 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-06-18T02:43:39.573 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-06-18T02:43:39.573 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-06-18T02:43:39.573 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-06-18T02:43:39.573 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-06-18T02:43:39.573 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-06-18T02:43:39.573 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-06-18T02:43:39.573 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-06-18T02:43:39.573 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-06-18T02:43:39.574 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-06-18T02:43:39.575 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-06-18T02:43:39.575 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-06-18T02:43:39.575 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-06-18T02:43:39.575 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-06-18T02:43:39.575 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-06-18T02:43:39.575 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-06-18T02:43:39.575 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-06-18T02:43:39.575 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-06-18T02:43:39.575 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-06-18T02:43:39.575 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-06-18T02:43:39.575 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-06-18T02:43:39.575 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=563.579524ms 2024-06-18T02:43:39.575 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-06-18T02:43:39.576 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-06-18T02:43:39.576 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-06-18T02:43:39.576 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-06-18T02:43:39.576 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-06-18T02:43:39.576 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-06-18T02:43:39.576 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-06-18T02:43:39.576 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-06-18T02:43:39.576 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-06-18T02:43:39.576 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-06-18T02:43:39.576 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:43:39+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-06-18T02:43:40.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:39 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:40.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:39 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:43:40.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:39 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:43:40.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:39 smithi163 ceph-mon[25094]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-06-18T02:43:40.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:39 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:40.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:39 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:43:40.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:39 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:43:40.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:39 smithi163 ceph-mon[29791]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-06-18T02:43:40.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:40.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:43:40.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:43:40.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:39 smithi186 ceph-mon[25944]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-06-18T02:43:41.254 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:41 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:41.254 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:41 smithi186 ceph-mon[25944]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-06-18T02:43:41.391 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:41 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:41.391 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:41 smithi163 ceph-mon[25094]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-06-18T02:43:41.391 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:41 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:41.391 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:41 smithi163 ceph-mon[29791]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-06-18T02:43:43.387 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:43 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:43.387 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:43 smithi163 ceph-mon[25094]: osdmap e64: 8 total, 8 up, 8 in 2024-06-18T02:43:43.387 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:43 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/4242629029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-06-18T02:43:43.387 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:43 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-06-18T02:43:43.387 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:43 smithi163 ceph-mon[25094]: pgmap v51: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 1.9 KiB/s wr, 3 op/s 2024-06-18T02:43:43.387 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:43 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:43.387 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:43 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:43.387 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:43 smithi163 ceph-mon[29791]: osdmap e64: 8 total, 8 up, 8 in 2024-06-18T02:43:43.388 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:43 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/4242629029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-06-18T02:43:43.388 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:43 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-06-18T02:43:43.388 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:43 smithi163 ceph-mon[29791]: pgmap v51: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 1.9 KiB/s wr, 3 op/s 2024-06-18T02:43:43.388 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:43 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:43.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:43:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:43:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:43:43.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:43 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:43.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:43 smithi186 ceph-mon[25944]: osdmap e64: 8 total, 8 up, 8 in 2024-06-18T02:43:43.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:43 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/4242629029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-06-18T02:43:43.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:43 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-06-18T02:43:43.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:43 smithi186 ceph-mon[25944]: pgmap v51: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 1.9 KiB/s wr, 3 op/s 2024-06-18T02:43:43.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:43 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:44.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:44 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-06-18T02:43:44.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:44 smithi163 ceph-mon[25094]: osdmap e65: 8 total, 8 up, 8 in 2024-06-18T02:43:44.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:44 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:44.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:44 smithi163 ceph-mon[25094]: Reconfiguring alertmanager.a (dependencies changed)... 2024-06-18T02:43:44.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:44 smithi163 ceph-mon[25094]: Reconfiguring daemon alertmanager.a on smithi163 2024-06-18T02:43:44.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:44 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-06-18T02:43:44.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:44 smithi163 ceph-mon[29791]: osdmap e65: 8 total, 8 up, 8 in 2024-06-18T02:43:44.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:44 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:44.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:44 smithi163 ceph-mon[29791]: Reconfiguring alertmanager.a (dependencies changed)... 2024-06-18T02:43:44.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:44 smithi163 ceph-mon[29791]: Reconfiguring daemon alertmanager.a on smithi163 2024-06-18T02:43:44.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:44 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-06-18T02:43:44.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:44 smithi186 ceph-mon[25944]: osdmap e65: 8 total, 8 up, 8 in 2024-06-18T02:43:44.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:44 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:44.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:44 smithi186 ceph-mon[25944]: Reconfiguring alertmanager.a (dependencies changed)... 2024-06-18T02:43:44.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:44 smithi186 ceph-mon[25944]: Reconfiguring daemon alertmanager.a on smithi163 2024-06-18T02:43:45.530 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:45 smithi163 ceph-mon[29791]: osdmap e66: 8 total, 8 up, 8 in 2024-06-18T02:43:45.530 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:45 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/4242629029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-06-18T02:43:45.530 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:45 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-06-18T02:43:45.530 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:45 smithi163 ceph-mon[29791]: pgmap v54: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-06-18T02:43:45.530 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:45 smithi163 ceph-mon[25094]: osdmap e66: 8 total, 8 up, 8 in 2024-06-18T02:43:45.530 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:45 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/4242629029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-06-18T02:43:45.530 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:45 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-06-18T02:43:45.530 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:45 smithi163 ceph-mon[25094]: pgmap v54: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-06-18T02:43:45.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:45 smithi186 ceph-mon[25944]: osdmap e66: 8 total, 8 up, 8 in 2024-06-18T02:43:45.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:45 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/4242629029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-06-18T02:43:45.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:45 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-06-18T02:43:45.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:45 smithi186 ceph-mon[25944]: pgmap v54: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-06-18T02:43:46.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:46 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-06-18T02:43:46.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:46 smithi186 ceph-mon[25944]: osdmap e67: 8 total, 8 up, 8 in 2024-06-18T02:43:46.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:46 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-06-18T02:43:46.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:46 smithi163 ceph-mon[29791]: osdmap e67: 8 total, 8 up, 8 in 2024-06-18T02:43:46.588 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:46 smithi163 systemd[1]: Stopping Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:43:46.589 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:46 smithi163 bash[49652]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager.a" found: no such container 2024-06-18T02:43:46.589 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:46 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-06-18T02:43:46.589 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:46 smithi163 ceph-mon[25094]: osdmap e67: 8 total, 8 up, 8 in 2024-06-18T02:43:46.872 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:46 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[48138]: level=info ts=2024-06-18T02:43:46.588Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-06-18T02:43:46.872 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:46 smithi163 podman[49664]: 2024-06-18 02:43:46.619506883 +0000 UTC m=+0.129143131 container died b13a058d80866bbd98ee493e4db0af7e0fcd0f1d1de84b565b7d36fc1a76ab98 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:47.147 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:46 smithi163 podman[49664]: 2024-06-18 02:43:46.963983896 +0000 UTC m=+0.473620142 container cleanup b13a058d80866bbd98ee493e4db0af7e0fcd0f1d1de84b565b7d36fc1a76ab98 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:47.148 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:46 smithi163 bash[49664]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a 2024-06-18T02:43:47.541 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-mon[29791]: osdmap e68: 8 total, 8 up, 8 in 2024-06-18T02:43:47.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-06-18T02:43:47.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-mon[29791]: pgmap v57: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-06-18T02:43:47.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-mon[25094]: osdmap e68: 8 total, 8 up, 8 in 2024-06-18T02:43:47.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-06-18T02:43:47.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-mon[25094]: pgmap v57: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-06-18T02:43:47.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 podman[49678]: 2024-06-18 02:43:47.148612761 +0000 UTC m=+0.523490983 container remove b13a058d80866bbd98ee493e4db0af7e0fcd0f1d1de84b565b7d36fc1a76ab98 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:47.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 podman[49678]: 2024-06-18 02:43:47.282656065 +0000 UTC m=+0.657534289 volume remove aea8072f16c068e6439d08377c373dc9ccbbecb8ae91465baeb06a1438b4cf13 2024-06-18T02:43:47.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 bash[49689]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager.a" found: no such container 2024-06-18T02:43:47.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-06-18T02:43:47.545 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 systemd[1]: Stopped Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:43:47.545 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 systemd[1]: Starting Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:43:47.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 podman[49751]: 2024-06-18 02:43:47.542346472 +0000 UTC m=+0.104615297 volume create 3997a7386d9a69d4098f3364fdec86392c81093900224c1df5c6c0dadd01e97b 2024-06-18T02:43:47.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 podman[49751]: 2024-06-18 02:43:47.454413974 +0000 UTC m=+0.016682766 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-06-18T02:43:47.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 podman[49751]: 2024-06-18 02:43:47.634480365 +0000 UTC m=+0.196749146 container create 4277853c77458b299a4347b266b69cd74bfe5f59a6365a10c7e12ddee13a832b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:47.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:47 smithi186 ceph-mon[25944]: osdmap e68: 8 total, 8 up, 8 in 2024-06-18T02:43:47.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:47 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-06-18T02:43:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:47 smithi186 ceph-mon[25944]: pgmap v57: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-06-18T02:43:48.292 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 podman[49751]: 2024-06-18 02:43:47.887232475 +0000 UTC m=+0.449501268 container init 4277853c77458b299a4347b266b69cd74bfe5f59a6365a10c7e12ddee13a832b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:48.292 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 podman[49751]: 2024-06-18 02:43:47.890060573 +0000 UTC m=+0.452329366 container start 4277853c77458b299a4347b266b69cd74bfe5f59a6365a10c7e12ddee13a832b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:48.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=info ts=2024-06-18T02:43:47.900Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-06-18T02:43:48.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=info ts=2024-06-18T02:43:47.900Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-06-18T02:43:48.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=info ts=2024-06-18T02:43:47.900Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.163 port=9094 2024-06-18T02:43:48.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=info ts=2024-06-18T02:43:47.901Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-06-18T02:43:48.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=info ts=2024-06-18T02:43:47.926Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-18T02:43:48.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=info ts=2024-06-18T02:43:47.926Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-18T02:43:48.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=info ts=2024-06-18T02:43:47.929Z caller=main.go:518 msg=Listening address=:9093 2024-06-18T02:43:48.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=info ts=2024-06-18T02:43:47.929Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-06-18T02:43:48.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 bash[49751]: 4277853c77458b299a4347b266b69cd74bfe5f59a6365a10c7e12ddee13a832b 2024-06-18T02:43:48.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:47 smithi163 systemd[1]: Started Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:43:48.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:48 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-06-18T02:43:48.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:48 smithi163 ceph-mon[29791]: osdmap e69: 8 total, 8 up, 8 in 2024-06-18T02:43:48.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:48 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-06-18T02:43:48.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:48 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:48.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:48 smithi163 ceph-mon[29791]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T02:43:48.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:48 smithi163 ceph-mon[29791]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T02:43:48.624 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:48 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-06-18T02:43:48.624 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:48 smithi163 ceph-mon[25094]: osdmap e69: 8 total, 8 up, 8 in 2024-06-18T02:43:48.624 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:48 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-06-18T02:43:48.624 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:48 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:48.624 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:48 smithi163 ceph-mon[25094]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T02:43:48.624 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:48 smithi163 ceph-mon[25094]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T02:43:48.812 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:48 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-06-18T02:43:48.812 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:48 smithi186 ceph-mon[25944]: osdmap e69: 8 total, 8 up, 8 in 2024-06-18T02:43:48.812 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:48 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-06-18T02:43:48.812 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:48 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:48.813 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:48 smithi186 ceph-mon[25944]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T02:43:48.813 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:48 smithi186 ceph-mon[25944]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T02:43:49.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:43:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:43:48] "GET /metrics HTTP/1.1" 200 195260 "" "Prometheus/2.33.4" 2024-06-18T02:43:49.396 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T02:43:49.397 INFO:teuthology.orchestra.run.smithi163.stdout: "id": "53c6d780-3cba-4f3c-83b0-6d48d6c64236", 2024-06-18T02:43:49.397 INFO:teuthology.orchestra.run.smithi163.stdout: "epoch": 1, 2024-06-18T02:43:49.397 INFO:teuthology.orchestra.run.smithi163.stdout: "predecessor_uuid": "42b7a9ac-6cda-42af-a912-214a7b109b03", 2024-06-18T02:43:49.397 INFO:teuthology.orchestra.run.smithi163.stdout: "sync_status": [], 2024-06-18T02:43:49.397 INFO:teuthology.orchestra.run.smithi163.stdout: "period_map": { 2024-06-18T02:43:49.397 INFO:teuthology.orchestra.run.smithi163.stdout: "id": "53c6d780-3cba-4f3c-83b0-6d48d6c64236", 2024-06-18T02:43:49.397 INFO:teuthology.orchestra.run.smithi163.stdout: "zonegroups": [ 2024-06-18T02:43:49.397 INFO:teuthology.orchestra.run.smithi163.stdout: { 2024-06-18T02:43:49.397 INFO:teuthology.orchestra.run.smithi163.stdout: "id": "8ba594d6-06fa-4b90-a7de-629262a6b495", 2024-06-18T02:43:49.397 INFO:teuthology.orchestra.run.smithi163.stdout: "name": "default", 2024-06-18T02:43:49.397 INFO:teuthology.orchestra.run.smithi163.stdout: "api_name": "default", 2024-06-18T02:43:49.398 INFO:teuthology.orchestra.run.smithi163.stdout: "is_master": "true", 2024-06-18T02:43:49.398 INFO:teuthology.orchestra.run.smithi163.stdout: "endpoints": [], 2024-06-18T02:43:49.398 INFO:teuthology.orchestra.run.smithi163.stdout: "hostnames": [], 2024-06-18T02:43:49.398 INFO:teuthology.orchestra.run.smithi163.stdout: "hostnames_s3website": [], 2024-06-18T02:43:49.398 INFO:teuthology.orchestra.run.smithi163.stdout: "master_zone": "a82b479e-2b6b-407c-b5ce-5177fe7a57d3", 2024-06-18T02:43:49.398 INFO:teuthology.orchestra.run.smithi163.stdout: "zones": [ 2024-06-18T02:43:49.398 INFO:teuthology.orchestra.run.smithi163.stdout: { 2024-06-18T02:43:49.398 INFO:teuthology.orchestra.run.smithi163.stdout: "id": "a82b479e-2b6b-407c-b5ce-5177fe7a57d3", 2024-06-18T02:43:49.398 INFO:teuthology.orchestra.run.smithi163.stdout: "name": "z", 2024-06-18T02:43:49.398 INFO:teuthology.orchestra.run.smithi163.stdout: "endpoints": [], 2024-06-18T02:43:49.398 INFO:teuthology.orchestra.run.smithi163.stdout: "log_meta": "false", 2024-06-18T02:43:49.398 INFO:teuthology.orchestra.run.smithi163.stdout: "log_data": "false", 2024-06-18T02:43:49.398 INFO:teuthology.orchestra.run.smithi163.stdout: "bucket_index_max_shards": 11, 2024-06-18T02:43:49.399 INFO:teuthology.orchestra.run.smithi163.stdout: "read_only": "false", 2024-06-18T02:43:49.399 INFO:teuthology.orchestra.run.smithi163.stdout: "tier_type": "", 2024-06-18T02:43:49.399 INFO:teuthology.orchestra.run.smithi163.stdout: "sync_from_all": "true", 2024-06-18T02:43:49.399 INFO:teuthology.orchestra.run.smithi163.stdout: "sync_from": [], 2024-06-18T02:43:49.399 INFO:teuthology.orchestra.run.smithi163.stdout: "redirect_zone": "" 2024-06-18T02:43:49.399 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T02:43:49.399 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2024-06-18T02:43:49.399 INFO:teuthology.orchestra.run.smithi163.stdout: "placement_targets": [ 2024-06-18T02:43:49.399 INFO:teuthology.orchestra.run.smithi163.stdout: { 2024-06-18T02:43:49.399 INFO:teuthology.orchestra.run.smithi163.stdout: "name": "default-placement", 2024-06-18T02:43:49.399 INFO:teuthology.orchestra.run.smithi163.stdout: "tags": [], 2024-06-18T02:43:49.399 INFO:teuthology.orchestra.run.smithi163.stdout: "storage_classes": [ 2024-06-18T02:43:49.399 INFO:teuthology.orchestra.run.smithi163.stdout: "STANDARD" 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: "default_placement": "default-placement", 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: "realm_id": "3e8b7a8b-90be-43d6-93da-cdb514a88b8d", 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: "sync_policy": { 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: "groups": [] 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: "short_zone_ids": [ 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: { 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: "key": "a82b479e-2b6b-407c-b5ce-5177fe7a57d3", 2024-06-18T02:43:49.400 INFO:teuthology.orchestra.run.smithi163.stdout: "val": 2309531152 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: "master_zonegroup": "8ba594d6-06fa-4b90-a7de-629262a6b495", 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: "master_zone": "a82b479e-2b6b-407c-b5ce-5177fe7a57d3", 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: "period_config": { 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: "bucket_quota": { 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: "enabled": false, 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: "check_on_raw": false, 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: "max_size": -1, 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: "max_size_kb": 0, 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: "max_objects": -1 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: "user_quota": { 2024-06-18T02:43:49.401 INFO:teuthology.orchestra.run.smithi163.stdout: "enabled": false, 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: "check_on_raw": false, 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: "max_size": -1, 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: "max_size_kb": 0, 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: "max_objects": -1 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: "user_ratelimit": { 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: "max_read_ops": 0, 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: "max_write_ops": 0, 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: "max_read_bytes": 0, 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: "max_write_bytes": 0, 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: "enabled": false 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: "bucket_ratelimit": { 2024-06-18T02:43:49.402 INFO:teuthology.orchestra.run.smithi163.stdout: "max_read_ops": 0, 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: "max_write_ops": 0, 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: "max_read_bytes": 0, 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: "max_write_bytes": 0, 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: "enabled": false 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: "anonymous_ratelimit": { 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: "max_read_ops": 0, 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: "max_write_ops": 0, 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: "max_read_bytes": 0, 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: "max_write_bytes": 0, 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: "enabled": false 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: "realm_id": "3e8b7a8b-90be-43d6-93da-cdb514a88b8d", 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: "realm_name": "r", 2024-06-18T02:43:49.403 INFO:teuthology.orchestra.run.smithi163.stdout: "realm_epoch": 2 2024-06-18T02:43:49.404 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T02:43:49.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:49 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-06-18T02:43:49.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:49 smithi163 ceph-mon[25094]: osdmap e70: 8 total, 8 up, 8 in 2024-06-18T02:43:49.708 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:49 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-06-18T02:43:49.708 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:49 smithi163 ceph-mon[25094]: pgmap v60: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-06-18T02:43:49.708 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:49 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-06-18T02:43:49.708 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:49 smithi163 ceph-mon[25094]: osdmap e71: 8 total, 8 up, 8 in 2024-06-18T02:43:49.708 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:49 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-06-18T02:43:49.708 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:49 smithi163 ceph-mon[29791]: osdmap e70: 8 total, 8 up, 8 in 2024-06-18T02:43:49.708 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:49 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-06-18T02:43:49.708 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:49 smithi163 ceph-mon[29791]: pgmap v60: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-06-18T02:43:49.708 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:49 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-06-18T02:43:49.708 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:49 smithi163 ceph-mon[29791]: osdmap e71: 8 total, 8 up, 8 in 2024-06-18T02:43:49.771 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:49 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-06-18T02:43:49.771 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:49 smithi186 ceph-mon[25944]: osdmap e70: 8 total, 8 up, 8 in 2024-06-18T02:43:49.771 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:49 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-06-18T02:43:49.771 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:49 smithi186 ceph-mon[25944]: pgmap v60: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-06-18T02:43:49.772 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:49 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/175844708' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-06-18T02:43:49.772 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:49 smithi186 ceph-mon[25944]: osdmap e71: 8 total, 8 up, 8 in 2024-06-18T02:43:50.042 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:49 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=info ts=2024-06-18T02:43:49.902Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000984911s 2024-06-18T02:43:50.143 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-06-18T02:43:51.298 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 systemd[1]: Stopping Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:43:51.298 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 bash[42604]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-18T02:43:51.440 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:51 smithi163 ceph-mon[29791]: pgmap v62: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:51.440 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:51 smithi163 ceph-mon[25094]: pgmap v62: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:51 smithi186 ceph-mon[25944]: pgmap v62: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:51.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:43:51.304Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-18T02:43:51.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:43:51.304Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-06-18T02:43:51.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:43:51.304Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-06-18T02:43:51.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:43:51.304Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-06-18T02:43:51.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:43:51.304Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-06-18T02:43:51.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:43:51.304Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-06-18T02:43:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:43:51.304Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-18T02:43:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:43:51.304Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-06-18T02:43:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:43:51.304Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-06-18T02:43:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:43:51.304Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-06-18T02:43:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:43:51.304Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-06-18T02:43:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[40978]: ts=2024-06-18T02:43:51.304Z caller=main.go:1066 level=info msg="See you next time!" 2024-06-18T02:43:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 podman[42616]: 2024-06-18 02:43:51.314402691 +0000 UTC m=+0.110903058 container died cdd53f400d156cbb0c10d7c76a0992c9b90187395815bf77ee1b14f9012314fc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:51.946 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 podman[42616]: 2024-06-18 02:43:51.679021543 +0000 UTC m=+0.475521897 container cleanup cdd53f400d156cbb0c10d7c76a0992c9b90187395815bf77ee1b14f9012314fc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:51.946 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 bash[42616]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a 2024-06-18T02:43:51.946 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 bash[42643]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-18T02:43:52.085 INFO:teuthology.orchestra.run.smithi163.stdout:Scheduled rgw.foo update... 2024-06-18T02:43:52.205 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 podman[42631]: 2024-06-18 02:43:51.946843119 +0000 UTC m=+0.633129589 container remove cdd53f400d156cbb0c10d7c76a0992c9b90187395815bf77ee1b14f9012314fc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:52.205 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-06-18T02:43:52.205 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 systemd[1]: Stopped Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:43:52.205 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:51 smithi186 systemd[1]: Starting Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:43:52.205 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 podman[42704]: 2024-06-18 02:43:52.082161984 +0000 UTC m=+0.018104680 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-06-18T02:43:52.483 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 podman[42704]: 2024-06-18 02:43:52.206069144 +0000 UTC m=+0.142011829 container create 782012f610d36a34c191f21a4693fb405c48246d7b04e0cd55724064640f5978 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:52.483 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 podman[42704]: 2024-06-18 02:43:52.400471202 +0000 UTC m=+0.336413890 container init 782012f610d36a34c191f21a4693fb405c48246d7b04e0cd55724064640f5978 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:52.483 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 podman[42704]: 2024-06-18 02:43:52.403244021 +0000 UTC m=+0.339186706 container start 782012f610d36a34c191f21a4693fb405c48246d7b04e0cd55724064640f5978 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:43:52.483 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.426Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-06-18T02:43:52.483 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.426Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-06-18T02:43:52.483 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.426Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-06-18T02:43:52.483 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.426Z caller=main.go:518 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi186 (none))" 2024-06-18T02:43:52.484 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.426Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-18T02:43:52.484 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.426Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-18T02:43:52.484 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.428Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-06-18T02:43:52.484 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.428Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-06-18T02:43:52.484 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.429Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-06-18T02:43:52.484 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.430Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-18T02:43:52.484 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.430Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.476µs 2024-06-18T02:43:52.484 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.430Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-18T02:43:52.484 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 bash[42704]: 782012f610d36a34c191f21a4693fb405c48246d7b04e0cd55724064640f5978 2024-06-18T02:43:52.664 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph osd pool create foo' 2024-06-18T02:43:52.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 systemd[1]: Started Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:43:52.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.641Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-06-18T02:43:52.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.641Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-06-18T02:43:52.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.641Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.271µs wal_replay_duration=210.797239ms total_replay_duration=210.839796ms 2024-06-18T02:43:52.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.643Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-06-18T02:43:52.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.643Z caller=main.go:947 level=info msg="TSDB started" 2024-06-18T02:43:52.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.643Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-18T02:43:52.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.664Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.603649ms db_storage=746ns remote_storage=1.237µs web_handler=298ns query_engine=740ns scrape=382.808µs scrape_sd=28.065µs notify=25.343µs notify_sd=10.15µs rules=19.844511ms 2024-06-18T02:43:52.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:43:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:43:52.664Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-06-18T02:43:53.285 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='client.14697 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:43:53.285 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: Saving service rgw.foo spec with placement count:2 2024-06-18T02:43:53.285 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.285 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.285 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T02:43:53.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T02:43:53.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.163:9093"}]: dispatch 2024-06-18T02:43:53.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.163:9093"}]: dispatch 2024-06-18T02:43:53.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:43:53.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:43:53.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.186:9095"}]: dispatch 2024-06-18T02:43:53.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.186:9095"}]: dispatch 2024-06-18T02:43:53.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T02:43:53.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T02:43:53.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.186:3000"}]: dispatch 2024-06-18T02:43:53.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.186:3000"}]: dispatch 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[29791]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='client.14697 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: Saving service rgw.foo spec with placement count:2 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.163:9093"}]: dispatch 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.163:9093"}]: dispatch 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.287 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:43:53.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:43:53.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.186:9095"}]: dispatch 2024-06-18T02:43:53.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.186:9095"}]: dispatch 2024-06-18T02:43:53.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T02:43:53.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T02:43:53.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.186:3000"}]: dispatch 2024-06-18T02:43:53.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.186:3000"}]: dispatch 2024-06-18T02:43:53.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:43:53.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:43:53.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:53 smithi163 ceph-mon[25094]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-06-18T02:43:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='client.14697 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:43:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: Saving service rgw.foo spec with placement count:2 2024-06-18T02:43:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T02:43:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.163:9093"}]: dispatch 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.163:9093"}]: dispatch 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.186:9095"}]: dispatch 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.186:9095"}]: dispatch 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.186:3000"}]: dispatch 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.186:3000"}]: dispatch 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:43:53.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:43:53.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:53 smithi186 ceph-mon[25944]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-06-18T02:43:55.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:55 smithi163 ceph-mon[29791]: pgmap v64: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 930 B/s rd, 930 B/s wr, 2 op/s 2024-06-18T02:43:55.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:55 smithi163 ceph-mon[25094]: pgmap v64: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 930 B/s rd, 930 B/s wr, 2 op/s 2024-06-18T02:43:55.386 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:55 smithi186 ceph-mon[25944]: pgmap v64: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 930 B/s rd, 930 B/s wr, 2 op/s 2024-06-18T02:43:56.305 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'foo' created 2024-06-18T02:43:56.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:56 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1336288185' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-06-18T02:43:56.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:56 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-06-18T02:43:56.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:56 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:56.492 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:56 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1336288185' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-06-18T02:43:56.492 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:56 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-06-18T02:43:56.492 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:56 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:56 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1336288185' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-06-18T02:43:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:56 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-06-18T02:43:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:56 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.191 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'rbd pool init foo' 2024-06-18T02:43:57.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-06-18T02:43:57.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: osdmap e72: 8 total, 8 up, 8 in 2024-06-18T02:43:57.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: Saving service rgw.foo spec with placement count:2 2024-06-18T02:43:57.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T02:43:57.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T02:43:57.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-18T02:43:57.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:43:57.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: Deploying daemon rgw.foo.smithi163.mvqwty on smithi163 2024-06-18T02:43:57.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[29791]: pgmap v66: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-06-18T02:43:57.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: osdmap e72: 8 total, 8 up, 8 in 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: Saving service rgw.foo spec with placement count:2 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: Deploying daemon rgw.foo.smithi163.mvqwty on smithi163 2024-06-18T02:43:57.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-mon[25094]: pgmap v66: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-06-18T02:43:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-06-18T02:43:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: osdmap e72: 8 total, 8 up, 8 in 2024-06-18T02:43:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: Saving service rgw.foo spec with placement count:2 2024-06-18T02:43:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T02:43:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T02:43:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-18T02:43:57.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:43:57.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:43:57.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: Deploying daemon rgw.foo.smithi163.mvqwty on smithi163 2024-06-18T02:43:57.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:57 smithi186 ceph-mon[25944]: pgmap v66: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-06-18T02:43:58.200 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:43:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=info ts=2024-06-18T02:43:57.905Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003886027s 2024-06-18T02:43:58.459 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:58 smithi163 ceph-mon[25094]: osdmap e73: 8 total, 8 up, 8 in 2024-06-18T02:43:58.460 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:58 smithi163 ceph-mon[29791]: osdmap e73: 8 total, 8 up, 8 in 2024-06-18T02:43:58.745 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:43:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:43:58] "GET /metrics HTTP/1.1" 200 195260 "" "Prometheus/2.33.4" 2024-06-18T02:43:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:58 smithi186 ceph-mon[25944]: osdmap e73: 8 total, 8 up, 8 in 2024-06-18T02:43:59.366 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:59 smithi163 ceph-mon[29791]: osdmap e74: 8 total, 8 up, 8 in 2024-06-18T02:43:59.366 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:43:59 smithi163 ceph-mon[29791]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:59.366 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:59 smithi163 ceph-mon[25094]: osdmap e74: 8 total, 8 up, 8 in 2024-06-18T02:43:59.366 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:43:59 smithi163 ceph-mon[25094]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:43:59.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:59 smithi186 ceph-mon[25944]: osdmap e74: 8 total, 8 up, 8 in 2024-06-18T02:43:59.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:43:59 smithi186 ceph-mon[25944]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:44:00.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:00 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1418040854' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-06-18T02:44:00.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:00 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-06-18T02:44:00.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:00 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:00.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:00 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1418040854' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-06-18T02:44:00.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:00 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-06-18T02:44:00.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:00 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:00.804 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:00 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1418040854' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-06-18T02:44:00.804 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:00 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-06-18T02:44:00.804 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:00 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:01.745 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:01 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-06-18T02:44:01.746 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:01 smithi186 ceph-mon[25944]: osdmap e75: 8 total, 8 up, 8 in 2024-06-18T02:44:01.746 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:01 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:01.746 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:01 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T02:44:01.746 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:01 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T02:44:01.746 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:01 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-18T02:44:01.746 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:01 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:01.746 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:01 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:01.746 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:01 smithi186 ceph-mon[25944]: Deploying daemon rgw.foo.smithi186.vxkxtx on smithi186 2024-06-18T02:44:01.746 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:01 smithi186 ceph-mon[25944]: pgmap v71: 161 pgs: 22 unknown, 139 active+clean; 582 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 2.7 KiB/s wr, 115 op/s 2024-06-18T02:44:01.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-06-18T02:44:01.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[29791]: osdmap e75: 8 total, 8 up, 8 in 2024-06-18T02:44:01.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:01.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T02:44:01.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T02:44:01.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-18T02:44:01.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[29791]: Deploying daemon rgw.foo.smithi186.vxkxtx on smithi186 2024-06-18T02:44:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[29791]: pgmap v71: 161 pgs: 22 unknown, 139 active+clean; 582 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 2.7 KiB/s wr, 115 op/s 2024-06-18T02:44:01.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-06-18T02:44:01.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[25094]: osdmap e75: 8 total, 8 up, 8 in 2024-06-18T02:44:01.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:01.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T02:44:01.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T02:44:01.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-18T02:44:01.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[25094]: Deploying daemon rgw.foo.smithi186.vxkxtx on smithi186 2024-06-18T02:44:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:01 smithi163 ceph-mon[25094]: pgmap v71: 161 pgs: 22 unknown, 139 active+clean; 582 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 2.7 KiB/s wr, 115 op/s 2024-06-18T02:44:02.414 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:02 smithi186 ceph-mon[25944]: osdmap e76: 8 total, 8 up, 8 in 2024-06-18T02:44:02.651 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:02 smithi163 ceph-mon[29791]: osdmap e76: 8 total, 8 up, 8 in 2024-06-18T02:44:02.652 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:02 smithi163 ceph-mon[25094]: osdmap e76: 8 total, 8 up, 8 in 2024-06-18T02:44:03.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch apply iscsi foo u p' 2024-06-18T02:44:03.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:44:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:44:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:03 smithi186 ceph-mon[25944]: osdmap e77: 8 total, 8 up, 8 in 2024-06-18T02:44:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:03 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:03 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:03 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:44:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:03 smithi186 ceph-mon[25944]: pgmap v74: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 7.0 KiB/s wr, 324 op/s 2024-06-18T02:44:03.619 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:03 smithi163 ceph-mon[29791]: osdmap e77: 8 total, 8 up, 8 in 2024-06-18T02:44:03.619 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:03 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:03.620 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:03 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:03.620 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:03 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:44:03.620 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:03 smithi163 ceph-mon[29791]: pgmap v74: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 7.0 KiB/s wr, 324 op/s 2024-06-18T02:44:03.620 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:03 smithi163 ceph-mon[25094]: osdmap e77: 8 total, 8 up, 8 in 2024-06-18T02:44:03.620 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:03 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:03.620 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:03 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:03.620 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:03 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:44:03.620 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:03 smithi163 ceph-mon[25094]: pgmap v74: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 7.0 KiB/s wr, 324 op/s 2024-06-18T02:44:05.318 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:05 smithi163 ceph-mon[29791]: pgmap v75: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 117 KiB/s rd, 5.5 KiB/s wr, 254 op/s 2024-06-18T02:44:05.319 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:05 smithi163 ceph-mon[25094]: pgmap v75: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 117 KiB/s rd, 5.5 KiB/s wr, 254 op/s 2024-06-18T02:44:05.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:05 smithi186 ceph-mon[25944]: pgmap v75: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 117 KiB/s rd, 5.5 KiB/s wr, 254 op/s 2024-06-18T02:44:05.644 INFO:teuthology.orchestra.run.smithi163.stdout:Scheduled iscsi.foo update... 2024-06-18T02:44:06.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:06 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:06.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:06 smithi163 ceph-mon[29791]: from='client.24655 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:44:06.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:06 smithi163 ceph-mon[29791]: Saving service iscsi.foo spec with placement count:1 2024-06-18T02:44:06.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:06 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:06.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:06 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:06.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:06 smithi163 ceph-mon[25094]: from='client.24655 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:44:06.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:06 smithi163 ceph-mon[25094]: Saving service iscsi.foo spec with placement count:1 2024-06-18T02:44:06.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:06 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:06.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:06 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:06.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:06 smithi186 ceph-mon[25944]: from='client.24655 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:44:06.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:06 smithi186 ceph-mon[25944]: Saving service iscsi.foo spec with placement count:1 2024-06-18T02:44:06.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:06 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:07.027 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'sleep 180' 2024-06-18T02:44:09.019 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:08 smithi163 ceph-mon[25094]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 264 KiB/s rd, 5.4 KiB/s wr, 471 op/s 2024-06-18T02:44:09.020 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:08 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:09.020 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:44:08 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:44:08] "GET /metrics HTTP/1.1" 200 197883 "" "Prometheus/2.33.4" 2024-06-18T02:44:09.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:08 smithi163 ceph-mon[29791]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 264 KiB/s rd, 5.4 KiB/s wr, 471 op/s 2024-06-18T02:44:09.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:08 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:09.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:08 smithi186 ceph-mon[25944]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 264 KiB/s rd, 5.4 KiB/s wr, 471 op/s 2024-06-18T02:44:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:08 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.249 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[29791]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 189 KiB/s rd, 2.9 KiB/s wr, 326 op/s 2024-06-18T02:44:11.249 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.249 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[29791]: osdmap e78: 8 total, 8 up, 8 in 2024-06-18T02:44:11.250 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.250 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.250 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.250 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.250 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:44:11.250 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:44:11.251 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-06-18T02:44:11.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[25094]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 189 KiB/s rd, 2.9 KiB/s wr, 326 op/s 2024-06-18T02:44:11.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[25094]: osdmap e78: 8 total, 8 up, 8 in 2024-06-18T02:44:11.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:44:11.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:44:11.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:10 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-06-18T02:44:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:10 smithi186 ceph-mon[25944]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 189 KiB/s rd, 2.9 KiB/s wr, 326 op/s 2024-06-18T02:44:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:10 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:10 smithi186 ceph-mon[25944]: osdmap e78: 8 total, 8 up, 8 in 2024-06-18T02:44:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:10 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:10 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:10 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:10 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:11.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:10 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:44:11.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:10 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:44:11.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:10 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-06-18T02:44:12.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:11 smithi186 ceph-mon[25944]: Checking dashboard <-> RGW credentials 2024-06-18T02:44:12.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:11 smithi186 ceph-mon[25944]: Checking pool "foo" exists for service iscsi.foo 2024-06-18T02:44:12.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:11 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:12.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:11 smithi186 ceph-mon[25944]: Deploying daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:44:12.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:11 smithi163 ceph-mon[25094]: Checking dashboard <-> RGW credentials 2024-06-18T02:44:12.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:11 smithi163 ceph-mon[25094]: Checking pool "foo" exists for service iscsi.foo 2024-06-18T02:44:12.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:11 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:12.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:11 smithi163 ceph-mon[25094]: Deploying daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:44:12.179 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:11 smithi163 ceph-mon[29791]: Checking dashboard <-> RGW credentials 2024-06-18T02:44:12.179 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:11 smithi163 ceph-mon[29791]: Checking pool "foo" exists for service iscsi.foo 2024-06-18T02:44:12.179 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:11 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:12.179 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:11 smithi163 ceph-mon[29791]: Deploying daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:44:13.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:12 smithi163 ceph-mon[29791]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 471 B/s wr, 197 op/s 2024-06-18T02:44:13.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:12 smithi163 ceph-mon[29791]: mgrmap e20: y(active, since 116s), standbys: x 2024-06-18T02:44:13.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:12 smithi163 ceph-mon[25094]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 471 B/s wr, 197 op/s 2024-06-18T02:44:13.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:12 smithi163 ceph-mon[25094]: mgrmap e20: y(active, since 116s), standbys: x 2024-06-18T02:44:13.193 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:12 smithi186 ceph-mon[25944]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 471 B/s wr, 197 op/s 2024-06-18T02:44:13.193 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:12 smithi186 ceph-mon[25944]: mgrmap e20: y(active, since 116s), standbys: x 2024-06-18T02:44:13.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:44:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:44:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[29791]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 511 B/s wr, 179 op/s 2024-06-18T02:44:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:44:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/425595248' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T02:44:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2669572188' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1760538181"}]: dispatch 2024-06-18T02:44:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1760538181"}]: dispatch 2024-06-18T02:44:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[25094]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 511 B/s wr, 179 op/s 2024-06-18T02:44:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:44:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/425595248' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T02:44:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2669572188' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1760538181"}]: dispatch 2024-06-18T02:44:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:14 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1760538181"}]: dispatch 2024-06-18T02:44:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:14 smithi186 ceph-mon[25944]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 511 B/s wr, 179 op/s 2024-06-18T02:44:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:14 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:14 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:14 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:44:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:14 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/425595248' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T02:44:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:14 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2669572188' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1760538181"}]: dispatch 2024-06-18T02:44:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:14 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1760538181"}]: dispatch 2024-06-18T02:44:16.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1760538181"}]': finished 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: osdmap e79: 8 total, 8 up, 8 in 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:44:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:16.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1420600409' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1226154455"}]: dispatch 2024-06-18T02:44:16.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1226154455"}]: dispatch 2024-06-18T02:44:16.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1760538181"}]': finished 2024-06-18T02:44:16.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: osdmap e79: 8 total, 8 up, 8 in 2024-06-18T02:44:16.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-06-18T02:44:16.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-06-18T02:44:16.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-06-18T02:44:16.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-06-18T02:44:16.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-06-18T02:44:16.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-06-18T02:44:16.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-06-18T02:44:16.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-06-18T02:44:16.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:44:16.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:44:16.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:44:16.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:44:16.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:16.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1420600409' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1226154455"}]: dispatch 2024-06-18T02:44:16.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:15 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1226154455"}]: dispatch 2024-06-18T02:44:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1760538181"}]': finished 2024-06-18T02:44:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: osdmap e79: 8 total, 8 up, 8 in 2024-06-18T02:44:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-06-18T02:44:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-06-18T02:44:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-06-18T02:44:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-06-18T02:44:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-06-18T02:44:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-06-18T02:44:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-06-18T02:44:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-06-18T02:44:16.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:44:16.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:44:16.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:44:16.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:44:16.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:16.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1420600409' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1226154455"}]: dispatch 2024-06-18T02:44:16.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:15 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1226154455"}]: dispatch 2024-06-18T02:44:17.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[25094]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 20 op/s 2024-06-18T02:44:17.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]': finished 2024-06-18T02:44:17.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-06-18T02:44:17.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-06-18T02:44:17.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-06-18T02:44:17.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1226154455"}]': finished 2024-06-18T02:44:17.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[25094]: osdmap e80: 8 total, 8 up, 8 in 2024-06-18T02:44:17.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2718332857' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3210434964"}]: dispatch 2024-06-18T02:44:17.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3210434964"}]: dispatch 2024-06-18T02:44:17.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[29791]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 20 op/s 2024-06-18T02:44:17.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]': finished 2024-06-18T02:44:17.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-06-18T02:44:17.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-06-18T02:44:17.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-06-18T02:44:17.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1226154455"}]': finished 2024-06-18T02:44:17.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[29791]: osdmap e80: 8 total, 8 up, 8 in 2024-06-18T02:44:17.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2718332857' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3210434964"}]: dispatch 2024-06-18T02:44:17.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:16 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3210434964"}]: dispatch 2024-06-18T02:44:17.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:16 smithi186 ceph-mon[25944]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 20 op/s 2024-06-18T02:44:17.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:16 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]': finished 2024-06-18T02:44:17.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:16 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-06-18T02:44:17.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:16 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-06-18T02:44:17.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:16 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-06-18T02:44:17.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:16 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1226154455"}]': finished 2024-06-18T02:44:17.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:16 smithi186 ceph-mon[25944]: osdmap e80: 8 total, 8 up, 8 in 2024-06-18T02:44:17.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:16 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2718332857' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3210434964"}]: dispatch 2024-06-18T02:44:17.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:16 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3210434964"}]: dispatch 2024-06-18T02:44:18.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3210434964"}]': finished 2024-06-18T02:44:18.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: osdmap e81: 8 total, 8 up, 8 in 2024-06-18T02:44:18.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:44:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:44:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi163"}]: dispatch 2024-06-18T02:44:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:44:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2687807836' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2324578116"}]: dispatch 2024-06-18T02:44:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2324578116"}]: dispatch 2024-06-18T02:44:18.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3210434964"}]': finished 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: osdmap e81: 8 total, 8 up, 8 in 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi163"}]: dispatch 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2687807836' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2324578116"}]: dispatch 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2324578116"}]: dispatch 2024-06-18T02:44:18.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:17 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3210434964"}]': finished 2024-06-18T02:44:18.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: osdmap e81: 8 total, 8 up, 8 in 2024-06-18T02:44:18.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:44:18.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:44:18.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi163"}]: dispatch 2024-06-18T02:44:18.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:18.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:44:18.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:44:18.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2687807836' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2324578116"}]: dispatch 2024-06-18T02:44:18.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2324578116"}]: dispatch 2024-06-18T02:44:18.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:17 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:44:19.024 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:44:18 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:44:18] "GET /metrics HTTP/1.1" 200 208140 "" "Prometheus/2.33.4" 2024-06-18T02:44:19.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[29791]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 341 B/s wr, 14 op/s 2024-06-18T02:44:19.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:44:19.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[29791]: Adding iSCSI gateway http://:@172.21.15.163:5000 to Dashboard 2024-06-18T02:44:19.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:44:19.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi163"}]: dispatch 2024-06-18T02:44:19.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[29791]: Checking dashboard <-> RGW credentials 2024-06-18T02:44:19.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2324578116"}]': finished 2024-06-18T02:44:19.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[29791]: osdmap e82: 8 total, 8 up, 8 in 2024-06-18T02:44:19.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1247557044' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/911896598"}]: dispatch 2024-06-18T02:44:19.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/911896598"}]: dispatch 2024-06-18T02:44:19.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[25094]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 341 B/s wr, 14 op/s 2024-06-18T02:44:19.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:44:19.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[25094]: Adding iSCSI gateway http://:@172.21.15.163:5000 to Dashboard 2024-06-18T02:44:19.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:44:19.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi163"}]: dispatch 2024-06-18T02:44:19.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[25094]: Checking dashboard <-> RGW credentials 2024-06-18T02:44:19.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2324578116"}]': finished 2024-06-18T02:44:19.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[25094]: osdmap e82: 8 total, 8 up, 8 in 2024-06-18T02:44:19.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1247557044' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/911896598"}]: dispatch 2024-06-18T02:44:19.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:19 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/911896598"}]: dispatch 2024-06-18T02:44:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:19 smithi186 ceph-mon[25944]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 341 B/s wr, 14 op/s 2024-06-18T02:44:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:19 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:44:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:19 smithi186 ceph-mon[25944]: Adding iSCSI gateway http://:@172.21.15.163:5000 to Dashboard 2024-06-18T02:44:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:19 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:44:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:19 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi163"}]: dispatch 2024-06-18T02:44:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:19 smithi186 ceph-mon[25944]: Checking dashboard <-> RGW credentials 2024-06-18T02:44:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:19 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2324578116"}]': finished 2024-06-18T02:44:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:19 smithi186 ceph-mon[25944]: osdmap e82: 8 total, 8 up, 8 in 2024-06-18T02:44:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:19 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1247557044' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/911896598"}]: dispatch 2024-06-18T02:44:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:19 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/911896598"}]: dispatch 2024-06-18T02:44:20.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:20 smithi186 ceph-mon[25944]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 736 B/s rd, 245 B/s wr, 2 op/s 2024-06-18T02:44:20.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:20 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/911896598"}]': finished 2024-06-18T02:44:20.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:20 smithi186 ceph-mon[25944]: osdmap e83: 8 total, 8 up, 8 in 2024-06-18T02:44:20.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:20 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2808711313' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/911896598"}]: dispatch 2024-06-18T02:44:20.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:20 smithi163 ceph-mon[29791]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 736 B/s rd, 245 B/s wr, 2 op/s 2024-06-18T02:44:20.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:20 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/911896598"}]': finished 2024-06-18T02:44:20.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:20 smithi163 ceph-mon[29791]: osdmap e83: 8 total, 8 up, 8 in 2024-06-18T02:44:20.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:20 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2808711313' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/911896598"}]: dispatch 2024-06-18T02:44:20.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:20 smithi163 ceph-mon[25094]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 736 B/s rd, 245 B/s wr, 2 op/s 2024-06-18T02:44:20.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:20 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/911896598"}]': finished 2024-06-18T02:44:20.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:20 smithi163 ceph-mon[25094]: osdmap e83: 8 total, 8 up, 8 in 2024-06-18T02:44:20.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:20 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2808711313' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/911896598"}]: dispatch 2024-06-18T02:44:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:21 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2808711313' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/911896598"}]': finished 2024-06-18T02:44:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:21 smithi186 ceph-mon[25944]: osdmap e84: 8 total, 8 up, 8 in 2024-06-18T02:44:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:21 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/87248245' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1291328983"}]: dispatch 2024-06-18T02:44:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:21 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1291328983"}]: dispatch 2024-06-18T02:44:21.327 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:21 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2808711313' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/911896598"}]': finished 2024-06-18T02:44:21.327 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:21 smithi163 ceph-mon[25094]: osdmap e84: 8 total, 8 up, 8 in 2024-06-18T02:44:21.327 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:21 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/87248245' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1291328983"}]: dispatch 2024-06-18T02:44:21.327 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:21 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1291328983"}]: dispatch 2024-06-18T02:44:21.327 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:21 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2808711313' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/911896598"}]': finished 2024-06-18T02:44:21.327 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:21 smithi163 ceph-mon[29791]: osdmap e84: 8 total, 8 up, 8 in 2024-06-18T02:44:21.328 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:21 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/87248245' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1291328983"}]: dispatch 2024-06-18T02:44:21.328 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:21 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1291328983"}]: dispatch 2024-06-18T02:44:22.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:22 smithi186 ceph-mon[25944]: pgmap v90: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 71 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 8 op/s 2024-06-18T02:44:22.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:22 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1291328983"}]': finished 2024-06-18T02:44:22.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:22 smithi186 ceph-mon[25944]: osdmap e85: 8 total, 8 up, 8 in 2024-06-18T02:44:22.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:22 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2385459877' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4217980061"}]: dispatch 2024-06-18T02:44:22.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:22 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4217980061"}]: dispatch 2024-06-18T02:44:22.572 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:22 smithi163 ceph-mon[29791]: pgmap v90: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 71 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 8 op/s 2024-06-18T02:44:22.573 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:22 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1291328983"}]': finished 2024-06-18T02:44:22.573 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:22 smithi163 ceph-mon[29791]: osdmap e85: 8 total, 8 up, 8 in 2024-06-18T02:44:22.573 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:22 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2385459877' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4217980061"}]: dispatch 2024-06-18T02:44:22.573 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:22 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4217980061"}]: dispatch 2024-06-18T02:44:22.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:22 smithi163 ceph-mon[25094]: pgmap v90: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 71 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 8 op/s 2024-06-18T02:44:22.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:22 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1291328983"}]': finished 2024-06-18T02:44:22.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:22 smithi163 ceph-mon[25094]: osdmap e85: 8 total, 8 up, 8 in 2024-06-18T02:44:22.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:22 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2385459877' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4217980061"}]: dispatch 2024-06-18T02:44:22.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:22 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4217980061"}]: dispatch 2024-06-18T02:44:23.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:44:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:44:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:44:23.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:23 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4217980061"}]': finished 2024-06-18T02:44:23.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:23 smithi186 ceph-mon[25944]: osdmap e86: 8 total, 8 up, 8 in 2024-06-18T02:44:23.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:23 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2374514201' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3911242098"}]: dispatch 2024-06-18T02:44:23.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:23 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3911242098"}]: dispatch 2024-06-18T02:44:23.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:23 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4217980061"}]': finished 2024-06-18T02:44:23.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:23 smithi163 ceph-mon[29791]: osdmap e86: 8 total, 8 up, 8 in 2024-06-18T02:44:23.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:23 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2374514201' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3911242098"}]: dispatch 2024-06-18T02:44:23.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:23 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3911242098"}]: dispatch 2024-06-18T02:44:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:23 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4217980061"}]': finished 2024-06-18T02:44:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:23 smithi163 ceph-mon[25094]: osdmap e86: 8 total, 8 up, 8 in 2024-06-18T02:44:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:23 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2374514201' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3911242098"}]: dispatch 2024-06-18T02:44:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:23 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3911242098"}]: dispatch 2024-06-18T02:44:24.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:24 smithi163 ceph-mon[29791]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-06-18T02:44:24.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:24 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3911242098"}]': finished 2024-06-18T02:44:24.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:24 smithi163 ceph-mon[29791]: osdmap e87: 8 total, 8 up, 8 in 2024-06-18T02:44:24.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:24 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2164533918' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/396937736"}]: dispatch 2024-06-18T02:44:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:24 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:44:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:24 smithi163 ceph-mon[25094]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-06-18T02:44:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:24 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3911242098"}]': finished 2024-06-18T02:44:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:24 smithi163 ceph-mon[25094]: osdmap e87: 8 total, 8 up, 8 in 2024-06-18T02:44:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:24 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2164533918' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/396937736"}]: dispatch 2024-06-18T02:44:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:24 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:44:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:24 smithi186 ceph-mon[25944]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-06-18T02:44:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:24 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3911242098"}]': finished 2024-06-18T02:44:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:24 smithi186 ceph-mon[25944]: osdmap e87: 8 total, 8 up, 8 in 2024-06-18T02:44:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:24 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2164533918' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/396937736"}]: dispatch 2024-06-18T02:44:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:24 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:44:25.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:25 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2164533918' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/396937736"}]': finished 2024-06-18T02:44:25.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:25 smithi163 ceph-mon[25094]: osdmap e88: 8 total, 8 up, 8 in 2024-06-18T02:44:25.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:25 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/648313166' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/396937736"}]: dispatch 2024-06-18T02:44:25.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:25 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2164533918' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/396937736"}]': finished 2024-06-18T02:44:25.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:25 smithi163 ceph-mon[29791]: osdmap e88: 8 total, 8 up, 8 in 2024-06-18T02:44:25.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:25 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/648313166' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/396937736"}]: dispatch 2024-06-18T02:44:25.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:25 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2164533918' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/396937736"}]': finished 2024-06-18T02:44:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:25 smithi186 ceph-mon[25944]: osdmap e88: 8 total, 8 up, 8 in 2024-06-18T02:44:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:25 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/648313166' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/396937736"}]: dispatch 2024-06-18T02:44:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:26 smithi163 ceph-mon[25094]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T02:44:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:26 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/648313166' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/396937736"}]': finished 2024-06-18T02:44:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:26 smithi163 ceph-mon[25094]: osdmap e89: 8 total, 8 up, 8 in 2024-06-18T02:44:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:26 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/470666617' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/921115307"}]: dispatch 2024-06-18T02:44:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:26 smithi163 ceph-mon[29791]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T02:44:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:26 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/648313166' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/396937736"}]': finished 2024-06-18T02:44:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:26 smithi163 ceph-mon[29791]: osdmap e89: 8 total, 8 up, 8 in 2024-06-18T02:44:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:26 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/470666617' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/921115307"}]: dispatch 2024-06-18T02:44:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:26 smithi186 ceph-mon[25944]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T02:44:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:26 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/648313166' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/396937736"}]': finished 2024-06-18T02:44:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:26 smithi186 ceph-mon[25944]: osdmap e89: 8 total, 8 up, 8 in 2024-06-18T02:44:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:26 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/470666617' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/921115307"}]: dispatch 2024-06-18T02:44:27.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:27 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/470666617' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/921115307"}]': finished 2024-06-18T02:44:27.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:27 smithi163 ceph-mon[29791]: osdmap e90: 8 total, 8 up, 8 in 2024-06-18T02:44:27.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:27 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1656169301' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3175087117"}]: dispatch 2024-06-18T02:44:27.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:27 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3175087117"}]: dispatch 2024-06-18T02:44:27.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:27 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/470666617' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/921115307"}]': finished 2024-06-18T02:44:27.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:27 smithi163 ceph-mon[25094]: osdmap e90: 8 total, 8 up, 8 in 2024-06-18T02:44:27.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:27 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1656169301' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3175087117"}]: dispatch 2024-06-18T02:44:27.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:27 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3175087117"}]: dispatch 2024-06-18T02:44:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:27 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/470666617' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/921115307"}]': finished 2024-06-18T02:44:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:27 smithi186 ceph-mon[25944]: osdmap e90: 8 total, 8 up, 8 in 2024-06-18T02:44:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:27 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1656169301' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3175087117"}]: dispatch 2024-06-18T02:44:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:27 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3175087117"}]: dispatch 2024-06-18T02:44:28.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:28 smithi163 ceph-mon[29791]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:28.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:28 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3175087117"}]': finished 2024-06-18T02:44:28.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:28 smithi163 ceph-mon[29791]: osdmap e91: 8 total, 8 up, 8 in 2024-06-18T02:44:28.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:28 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1383285034' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2926912200"}]: dispatch 2024-06-18T02:44:28.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:28 smithi163 ceph-mon[25094]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:28.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:28 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3175087117"}]': finished 2024-06-18T02:44:28.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:28 smithi163 ceph-mon[25094]: osdmap e91: 8 total, 8 up, 8 in 2024-06-18T02:44:28.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:28 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1383285034' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2926912200"}]: dispatch 2024-06-18T02:44:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:28 smithi186 ceph-mon[25944]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:28 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3175087117"}]': finished 2024-06-18T02:44:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:28 smithi186 ceph-mon[25944]: osdmap e91: 8 total, 8 up, 8 in 2024-06-18T02:44:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:28 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1383285034' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2926912200"}]: dispatch 2024-06-18T02:44:29.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:44:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:44:28] "GET /metrics HTTP/1.1" 200 208140 "" "Prometheus/2.33.4" 2024-06-18T02:44:29.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:29 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1383285034' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2926912200"}]': finished 2024-06-18T02:44:29.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:29 smithi163 ceph-mon[29791]: osdmap e92: 8 total, 8 up, 8 in 2024-06-18T02:44:29.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:29 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3276993210' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4217980061"}]: dispatch 2024-06-18T02:44:29.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:29 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4217980061"}]: dispatch 2024-06-18T02:44:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:29 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1383285034' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2926912200"}]': finished 2024-06-18T02:44:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:29 smithi163 ceph-mon[25094]: osdmap e92: 8 total, 8 up, 8 in 2024-06-18T02:44:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:29 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3276993210' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4217980061"}]: dispatch 2024-06-18T02:44:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:29 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4217980061"}]: dispatch 2024-06-18T02:44:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:29 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1383285034' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2926912200"}]': finished 2024-06-18T02:44:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:29 smithi186 ceph-mon[25944]: osdmap e92: 8 total, 8 up, 8 in 2024-06-18T02:44:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:29 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3276993210' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4217980061"}]: dispatch 2024-06-18T02:44:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:29 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4217980061"}]: dispatch 2024-06-18T02:44:30.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:30 smithi163 ceph-mon[29791]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:30.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:30 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4217980061"}]': finished 2024-06-18T02:44:30.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:30 smithi163 ceph-mon[29791]: osdmap e93: 8 total, 8 up, 8 in 2024-06-18T02:44:30.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:30 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3503368247' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/538097455"}]: dispatch 2024-06-18T02:44:30.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:30 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/538097455"}]: dispatch 2024-06-18T02:44:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:30 smithi163 ceph-mon[25094]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:30 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4217980061"}]': finished 2024-06-18T02:44:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:30 smithi163 ceph-mon[25094]: osdmap e93: 8 total, 8 up, 8 in 2024-06-18T02:44:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:30 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3503368247' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/538097455"}]: dispatch 2024-06-18T02:44:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:30 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/538097455"}]: dispatch 2024-06-18T02:44:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:30 smithi186 ceph-mon[25944]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:30 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4217980061"}]': finished 2024-06-18T02:44:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:30 smithi186 ceph-mon[25944]: osdmap e93: 8 total, 8 up, 8 in 2024-06-18T02:44:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:30 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3503368247' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/538097455"}]: dispatch 2024-06-18T02:44:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:30 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/538097455"}]: dispatch 2024-06-18T02:44:31.719 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:31 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/538097455"}]': finished 2024-06-18T02:44:31.719 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:31 smithi186 ceph-mon[25944]: osdmap e94: 8 total, 8 up, 8 in 2024-06-18T02:44:31.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:31 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/538097455"}]': finished 2024-06-18T02:44:31.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:31 smithi163 ceph-mon[29791]: osdmap e94: 8 total, 8 up, 8 in 2024-06-18T02:44:31.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:31 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/538097455"}]': finished 2024-06-18T02:44:31.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:31 smithi163 ceph-mon[25094]: osdmap e94: 8 total, 8 up, 8 in 2024-06-18T02:44:32.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:32 smithi186 ceph-mon[25944]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:32.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:32 smithi163 ceph-mon[29791]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:32.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:32 smithi163 ceph-mon[25094]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:33.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:44:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:44:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:44:34.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:34 smithi163 ceph-mon[29791]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 0 op/s 2024-06-18T02:44:34.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:34 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:44:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:34 smithi163 ceph-mon[25094]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 0 op/s 2024-06-18T02:44:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:34 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:44:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:34 smithi186 ceph-mon[25944]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 0 op/s 2024-06-18T02:44:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:34 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:44:36.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:36 smithi163 ceph-mon[25094]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 765 B/s rd, 0 op/s 2024-06-18T02:44:36.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:36 smithi163 ceph-mon[29791]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 765 B/s rd, 0 op/s 2024-06-18T02:44:36.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:36 smithi186 ceph-mon[25944]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 765 B/s rd, 0 op/s 2024-06-18T02:44:38.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:38 smithi163 ceph-mon[25094]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:38.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:44:38 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:44:38] "GET /metrics HTTP/1.1" 200 208169 "" "Prometheus/2.33.4" 2024-06-18T02:44:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:38 smithi163 ceph-mon[29791]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:38.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:38 smithi186 ceph-mon[25944]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:40.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:40 smithi163 ceph-mon[25094]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T02:44:40.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:40 smithi163 ceph-mon[29791]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T02:44:40.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:40 smithi186 ceph-mon[25944]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T02:44:42.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:42 smithi163 ceph-mon[29791]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T02:44:42.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:42 smithi163 ceph-mon[25094]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T02:44:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:42 smithi186 ceph-mon[25944]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T02:44:43.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:44:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:44:44.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:44 smithi163 ceph-mon[29791]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:44.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:44 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:44:44.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:44 smithi163 ceph-mon[25094]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:44 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:44:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:44 smithi186 ceph-mon[25944]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:44 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:44:46.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:46 smithi163 ceph-mon[25094]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:46.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:46 smithi163 ceph-mon[29791]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:46 smithi186 ceph-mon[25944]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:48.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:48 smithi163 ceph-mon[25094]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:48.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:48 smithi163 ceph-mon[29791]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:48.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:44:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:44:48] "GET /metrics HTTP/1.1" 200 208116 "" "Prometheus/2.33.4" 2024-06-18T02:44:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:48 smithi186 ceph-mon[25944]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:50.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:50 smithi163 ceph-mon[29791]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:50.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:50 smithi163 ceph-mon[25094]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:50.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:50 smithi186 ceph-mon[25944]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:52.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:52 smithi163 ceph-mon[29791]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:52.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:52 smithi163 ceph-mon[25094]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:52.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:52 smithi186 ceph-mon[25944]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:53.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:44:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:44:54.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:54 smithi163 ceph-mon[25094]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:54.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:54 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:44:54.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:54 smithi163 ceph-mon[29791]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:54.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:54 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:44:54.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:54 smithi186 ceph-mon[25944]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:54 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:44:56.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:56 smithi163 ceph-mon[25094]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:56.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:56 smithi163 ceph-mon[29791]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:56 smithi186 ceph-mon[25944]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:44:58.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:44:58 smithi163 ceph-mon[29791]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:58.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:44:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:44:58] "GET /metrics HTTP/1.1" 200 208116 "" "Prometheus/2.33.4" 2024-06-18T02:44:58.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:44:58 smithi163 ceph-mon[25094]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:44:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:44:58 smithi186 ceph-mon[25944]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:00.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:00 smithi163 ceph-mon[29791]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:00.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:00 smithi163 ceph-mon[25094]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:00.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:00 smithi186 ceph-mon[25944]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:02.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:02 smithi163 ceph-mon[25094]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:02.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:02 smithi163 ceph-mon[29791]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:02.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:02 smithi186 ceph-mon[25944]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:03.567 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:45:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:45:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:45:04.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:04 smithi163 ceph-mon[29791]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:04.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:04 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:04.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:04 smithi163 ceph-mon[25094]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:04.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:04 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:04.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:04 smithi186 ceph-mon[25944]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:04.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:04 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:06.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:06 smithi163 ceph-mon[25094]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:06.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:06 smithi163 ceph-mon[29791]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:06 smithi186 ceph-mon[25944]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:08.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:08 smithi163 ceph-mon[25094]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:08 smithi163 ceph-mon[29791]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:08.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:45:08 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:45:08] "GET /metrics HTTP/1.1" 200 208080 "" "Prometheus/2.33.4" 2024-06-18T02:45:08.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:08 smithi186 ceph-mon[25944]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:10.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:10 smithi163 ceph-mon[29791]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:10.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:10 smithi163 ceph-mon[25094]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:10.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:10 smithi186 ceph-mon[25944]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:12.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:12 smithi163 ceph-mon[29791]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:12.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:12 smithi163 ceph-mon[25094]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:12 smithi186 ceph-mon[25944]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:13.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:45:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:45:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:45:14.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:14 smithi163 ceph-mon[25094]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:14 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:14 smithi163 ceph-mon[29791]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:14 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:14 smithi186 ceph-mon[25944]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:14 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:15.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:45:15.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:45:15.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:45:15.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:45:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:45:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:45:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:45:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:45:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:45:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:45:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:45:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:45:16.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:16 smithi163 ceph-mon[25094]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:16.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:16 smithi163 ceph-mon[29791]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:16.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:16 smithi186 ceph-mon[25944]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:17.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:17 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:45:17.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:17 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:45:17.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:17 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:45:17.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:17 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:45:17.555 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:17 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:45:17.555 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:17 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:45:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:18 smithi186 ceph-mon[25944]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:18 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:45:19.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:18 smithi163 ceph-mon[29791]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:19.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:18 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:45:19.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:18 smithi163 ceph-mon[25094]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:19.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:18 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:45:19.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:45:18 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:45:18] "GET /metrics HTTP/1.1" 200 208064 "" "Prometheus/2.33.4" 2024-06-18T02:45:21.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:20 smithi163 ceph-mon[29791]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:21.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:20 smithi163 ceph-mon[25094]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:21.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:20 smithi186 ceph-mon[25944]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:22.844 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:22 smithi163 ceph-mon[25094]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:22.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:22 smithi163 ceph-mon[29791]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:23.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:22 smithi186 ceph-mon[25944]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:23.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:45:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:45:25.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:24 smithi163 ceph-mon[25094]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:25.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:24 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:25.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:24 smithi163 ceph-mon[29791]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:25.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:24 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:25.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:24 smithi186 ceph-mon[25944]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:25.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:24 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:27.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:26 smithi163 ceph-mon[29791]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:27.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:26 smithi163 ceph-mon[25094]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:27.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:26 smithi186 ceph-mon[25944]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:29.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:28 smithi163 ceph-mon[29791]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:29.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:28 smithi163 ceph-mon[25094]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:29.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:45:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:45:28] "GET /metrics HTTP/1.1" 200 208064 "" "Prometheus/2.33.4" 2024-06-18T02:45:29.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:28 smithi186 ceph-mon[25944]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:31.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:30 smithi163 ceph-mon[29791]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:31.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:30 smithi163 ceph-mon[25094]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:31.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:30 smithi186 ceph-mon[25944]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:32.870 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:32 smithi186 ceph-mon[25944]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:33.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:32 smithi163 ceph-mon[29791]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:33.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:32 smithi163 ceph-mon[25094]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:33.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:45:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:45:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:45:35.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:34 smithi163 ceph-mon[29791]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:35.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:34 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:35.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:34 smithi163 ceph-mon[25094]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:35.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:34 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:35.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:34 smithi186 ceph-mon[25944]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:35.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:34 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:37.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:36 smithi163 ceph-mon[29791]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:37.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:36 smithi163 ceph-mon[25094]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:37.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:36 smithi186 ceph-mon[25944]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:39.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:38 smithi163 ceph-mon[29791]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:39.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:38 smithi163 ceph-mon[25094]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:39.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:45:38 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:45:38] "GET /metrics HTTP/1.1" 200 208065 "" "Prometheus/2.33.4" 2024-06-18T02:45:39.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:38 smithi186 ceph-mon[25944]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:41.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:40 smithi163 ceph-mon[29791]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:41.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:40 smithi163 ceph-mon[25094]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:41.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:40 smithi186 ceph-mon[25944]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:43.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:42 smithi163 ceph-mon[25094]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:43.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:42 smithi163 ceph-mon[29791]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:43.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:42 smithi186 ceph-mon[25944]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:43.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:45:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:45:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:45:45.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:44 smithi163 ceph-mon[29791]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:45.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:44 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:45.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:44 smithi163 ceph-mon[25094]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:45.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:44 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:44 smithi186 ceph-mon[25944]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:44 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:47.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:46 smithi163 ceph-mon[29791]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:47.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:46 smithi163 ceph-mon[25094]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:47.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:46 smithi186 ceph-mon[25944]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:49.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:48 smithi163 ceph-mon[29791]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:49.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:45:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:45:48] "GET /metrics HTTP/1.1" 200 208062 "" "Prometheus/2.33.4" 2024-06-18T02:45:49.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:48 smithi163 ceph-mon[25094]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:49.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:48 smithi186 ceph-mon[25944]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:51.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:50 smithi163 ceph-mon[29791]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:51.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:50 smithi163 ceph-mon[25094]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:51.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:50 smithi186 ceph-mon[25944]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:52 smithi163 ceph-mon[29791]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:53.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:52 smithi163 ceph-mon[25094]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:52 smithi186 ceph-mon[25944]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:53.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:45:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:45:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:45:55.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:54 smithi163 ceph-mon[25094]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:55.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:54 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:55.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:54 smithi163 ceph-mon[29791]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:55.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:54 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:55.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:54 smithi186 ceph-mon[25944]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:55.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:54 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:45:57.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:56 smithi163 ceph-mon[25094]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:57.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:56 smithi163 ceph-mon[29791]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:57.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:56 smithi186 ceph-mon[25944]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:45:59.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:45:58 smithi163 ceph-mon[25094]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:59.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:45:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:45:58] "GET /metrics HTTP/1.1" 200 208062 "" "Prometheus/2.33.4" 2024-06-18T02:45:59.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:45:58 smithi163 ceph-mon[29791]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:45:59.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:45:58 smithi186 ceph-mon[25944]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:01.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:00 smithi163 ceph-mon[29791]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:01.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:00 smithi163 ceph-mon[25094]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:01.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:00 smithi186 ceph-mon[25944]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:03.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:02 smithi163 ceph-mon[25094]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:03.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:02 smithi163 ceph-mon[29791]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:02 smithi186 ceph-mon[25944]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:03.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:46:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:46:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:46:05.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:04 smithi163 ceph-mon[29791]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:05.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:04 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:04 smithi163 ceph-mon[25094]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:04 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:04 smithi186 ceph-mon[25944]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:04 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:07.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:06 smithi163 ceph-mon[29791]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:07.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:06 smithi163 ceph-mon[25094]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:07.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:06 smithi186 ceph-mon[25944]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:09.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:08 smithi163 ceph-mon[29791]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:09.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:08 smithi163 ceph-mon[25094]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:09.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:46:08 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:46:08] "GET /metrics HTTP/1.1" 200 208090 "" "Prometheus/2.33.4" 2024-06-18T02:46:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:08 smithi186 ceph-mon[25944]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:11.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:10 smithi163 ceph-mon[29791]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:11.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:10 smithi163 ceph-mon[25094]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:11.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:10 smithi186 ceph-mon[25944]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:13.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:12 smithi163 ceph-mon[29791]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:13.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:12 smithi163 ceph-mon[25094]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:13.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:12 smithi186 ceph-mon[25944]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:13.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:46:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:46:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:46:15.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:14 smithi163 ceph-mon[29791]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:15.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:14 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:15.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:14 smithi163 ceph-mon[25094]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:15.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:14 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:14 smithi186 ceph-mon[25944]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:14 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:16.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:46:16.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:46:16.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:46:16.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:46:16.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:46:16.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:46:16.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:46:16.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:46:16.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:46:16.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:46:16.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:46:16.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:46:17.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:16 smithi163 ceph-mon[29791]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:17.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:16 smithi163 ceph-mon[25094]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:17.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:16 smithi186 ceph-mon[25944]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:17.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:17 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:46:17.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:17 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:46:17.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:17 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:46:17.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:17 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:46:17.876 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:17 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:46:17.876 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:17 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:46:18.884 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:46:18 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:46:18] "GET /metrics HTTP/1.1" 200 208073 "" "Prometheus/2.33.4" 2024-06-18T02:46:19.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:18 smithi163 ceph-mon[29791]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:19.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:18 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:46:19.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:18 smithi163 ceph-mon[25094]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:19.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:18 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:46:19.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:18 smithi186 ceph-mon[25944]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:18 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:46:21.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:20 smithi163 ceph-mon[29791]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:21.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:20 smithi163 ceph-mon[25094]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:21.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:20 smithi186 ceph-mon[25944]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:23.192 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:22 smithi186 ceph-mon[25944]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:23.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:22 smithi163 ceph-mon[29791]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:23.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:22 smithi163 ceph-mon[25094]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:23.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:46:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:46:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:46:25.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:24 smithi163 ceph-mon[25094]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:25.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:24 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:25.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:24 smithi163 ceph-mon[29791]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:25.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:24 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:25.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:24 smithi186 ceph-mon[25944]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:25.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:24 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:27.273 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:26 smithi163 ceph-mon[29791]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:27.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:26 smithi163 ceph-mon[25094]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:27.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:26 smithi186 ceph-mon[25944]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:28.899 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:46:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:46:28] "GET /metrics HTTP/1.1" 200 208073 "" "Prometheus/2.33.4" 2024-06-18T02:46:29.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:28 smithi163 ceph-mon[29791]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:28 smithi163 ceph-mon[25094]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:29.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:28 smithi186 ceph-mon[25944]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:31.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:30 smithi163 ceph-mon[29791]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:31.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:30 smithi163 ceph-mon[25094]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:31.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:30 smithi186 ceph-mon[25944]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:33.192 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:32 smithi186 ceph-mon[25944]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:33.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:32 smithi163 ceph-mon[29791]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:33.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:32 smithi163 ceph-mon[25094]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:33.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:46:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:46:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:46:35.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:34 smithi163 ceph-mon[29791]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:35.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:34 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:35.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:34 smithi163 ceph-mon[25094]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:35.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:34 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:35.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:34 smithi186 ceph-mon[25944]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:35.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:34 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:37.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:36 smithi163 ceph-mon[29791]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:37.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:36 smithi163 ceph-mon[25094]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:37.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:36 smithi186 ceph-mon[25944]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:38.913 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:46:38 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:46:38] "GET /metrics HTTP/1.1" 200 208058 "" "Prometheus/2.33.4" 2024-06-18T02:46:39.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:38 smithi163 ceph-mon[29791]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:39.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:38 smithi163 ceph-mon[25094]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:39.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:38 smithi186 ceph-mon[25944]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:41.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:40 smithi163 ceph-mon[29791]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:41.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:40 smithi163 ceph-mon[25094]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:41.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:40 smithi186 ceph-mon[25944]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:43.192 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:42 smithi186 ceph-mon[25944]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:43.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:42 smithi163 ceph-mon[29791]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:43.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:42 smithi163 ceph-mon[25094]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:43.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:46:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:46:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:46:45.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:44 smithi163 ceph-mon[29791]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:45.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:44 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:45.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:44 smithi163 ceph-mon[25094]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:45.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:44 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:45.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:44 smithi186 ceph-mon[25944]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:45.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:44 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:47.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:46 smithi163 ceph-mon[29791]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:47.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:46 smithi163 ceph-mon[25094]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:47.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:46 smithi186 ceph-mon[25944]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:48.927 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:46:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:46:48] "GET /metrics HTTP/1.1" 200 208021 "" "Prometheus/2.33.4" 2024-06-18T02:46:49.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:48 smithi163 ceph-mon[29791]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:49.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:48 smithi163 ceph-mon[25094]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:49.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:48 smithi186 ceph-mon[25944]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:51.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:50 smithi163 ceph-mon[25094]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:50 smithi163 ceph-mon[29791]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:51.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:50 smithi186 ceph-mon[25944]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:53.193 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:52 smithi186 ceph-mon[25944]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:53.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:52 smithi163 ceph-mon[25094]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:53.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:52 smithi163 ceph-mon[29791]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:53.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:46:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:46:55.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:54 smithi163 ceph-mon[29791]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:55.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:54 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:55.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:54 smithi163 ceph-mon[25094]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:55.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:54 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:55.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:54 smithi186 ceph-mon[25944]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:55.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:54 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:46:57.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:56 smithi163 ceph-mon[25094]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:57.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:56 smithi163 ceph-mon[29791]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:57.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:56 smithi186 ceph-mon[25944]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:46:58.941 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:46:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:46:58] "GET /metrics HTTP/1.1" 200 208021 "" "Prometheus/2.33.4" 2024-06-18T02:46:59.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:46:58 smithi163 ceph-mon[29791]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:46:58 smithi163 ceph-mon[25094]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:46:59.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:46:58 smithi186 ceph-mon[25944]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:01.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:00 smithi163 ceph-mon[29791]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T02:47:01.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:00 smithi163 ceph-mon[25094]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T02:47:01.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:00 smithi186 ceph-mon[25944]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T02:47:03.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:02 smithi163 ceph-mon[29791]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:03.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:02 smithi163 ceph-mon[25094]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:03.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:02 smithi186 ceph-mon[25944]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:03.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:47:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:47:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:47:05.280 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:04 smithi186 ceph-mon[25944]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T02:47:05.280 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:04 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:05.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:04 smithi163 ceph-mon[29791]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T02:47:05.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:04 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:05.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:04 smithi163 ceph-mon[25094]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T02:47:05.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:04 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:07.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:06 smithi163 ceph-mon[29791]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T02:47:07.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:06 smithi163 ceph-mon[25094]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T02:47:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:06 smithi186 ceph-mon[25944]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T02:47:08.956 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:47:08 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:47:08] "GET /metrics HTTP/1.1" 200 208009 "" "Prometheus/2.33.4" 2024-06-18T02:47:09.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:08 smithi163 ceph-mon[29791]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:08 smithi163 ceph-mon[25094]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:08 smithi186 ceph-mon[25944]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:10.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-06-18T02:47:11.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:10 smithi163 ceph-mon[29791]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:11.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:10 smithi163 ceph-mon[25094]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:10 smithi186 ceph-mon[25944]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:12.691 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-06-18T02:47:13.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:12 smithi163 ceph-mon[25094]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:13.223 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:12 smithi163 ceph-mon[29791]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:13.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:12 smithi186 ceph-mon[25944]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:13.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:47:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:47:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:47:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:14 smithi186 ceph-mon[25944]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:14 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:15.247 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:14 smithi163 ceph-mon[29791]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:15.247 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:14 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:15.247 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:14 smithi163 ceph-mon[25094]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:15.247 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:14 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:15.270 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph config set global log_to_journald false --force' 2024-06-18T02:47:16.237 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:47:16.237 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:47:16.237 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:47:16.238 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:47:16.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:47:16.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:47:16.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:47:16.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:47:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:47:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:47:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:47:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:47:17.202 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:16 smithi163 ceph-mon[29791]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:17.202 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:16 smithi163 ceph-mon[25094]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:17.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:16 smithi186 ceph-mon[25944]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:17.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps' 2024-06-18T02:47:18.162 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:17 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:47:18.162 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:17 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:47:18.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:17 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:47:18.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:17 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:47:18.194 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:17 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:47:18.194 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:17 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:47:18.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:47:18 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:47:18] "GET /metrics HTTP/1.1" 200 208018 "" "Prometheus/2.33.4" 2024-06-18T02:47:19.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:18 smithi163 ceph-mon[25094]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:19.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:18 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:47:19.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:18 smithi163 ceph-mon[29791]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:19.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:18 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:47:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:18 smithi186 ceph-mon[25944]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:18 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:47:19.843 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T02:47:19.843 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (3m) 3m ago 3m 15.4M - ba2b418f427c 4277853c7745 2024-06-18T02:47:19.843 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (3m) 3m ago 3m 44.0M - 8.3.5 dad864ee21e9 f134c5a8fb6d 2024-06-18T02:47:19.843 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 running (3m) 3m ago 3m 57.4M - 3.5 e1d6a67b021e c28d1b6f808f 2024-06-18T02:47:19.843 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e d7519f47d8b0 2024-06-18T02:47:19.843 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:9283 running (10m) 3m ago 10m 453M - 17.2.0 e1d6a67b021e 256bee9cc14e 2024-06-18T02:47:19.843 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (10m) 3m ago 10m 49.6M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T02:47:19.843 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (8m) 3m ago 8m 39.4M 2048M 17.2.0 e1d6a67b021e a3618f734a9f 2024-06-18T02:47:19.843 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (8m) 3m ago 8m 38.8M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T02:47:19.843 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (4m) 3m ago 4m 16.5M - 1dbe0e931976 4ed4364c9a98 2024-06-18T02:47:19.843 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (4m) 3m ago 4m 18.7M - 1dbe0e931976 48a19a288113 2024-06-18T02:47:19.844 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (8m) 3m ago 8m 49.6M 4016M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T02:47:19.844 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (7m) 3m ago 7m 49.6M 4016M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T02:47:19.844 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (7m) 3m ago 7m 44.2M 4016M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T02:47:19.844 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (7m) 3m ago 7m 45.7M 4016M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T02:47:19.844 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (6m) 3m ago 6m 52.0M 4272M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T02:47:19.844 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (6m) 3m ago 6m 44.3M 4272M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T02:47:19.844 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (5m) 3m ago 5m 43.6M 4272M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T02:47:19.844 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (5m) 3m ago 5m 45.8M 4272M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T02:47:19.844 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (3m) 3m ago 4m 44.2M - 514e6a882f6e 782012f610d3 2024-06-18T02:47:19.844 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (3m) 3m ago 3m 72.2M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T02:47:19.844 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (3m) 3m ago 3m 72.3M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T02:47:20.479 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions' 2024-06-18T02:47:21.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:20 smithi163 ceph-mon[29791]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:21.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:20 smithi163 ceph-mon[29791]: from='client.24880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:47:21.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:20 smithi163 ceph-mon[25094]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:21.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:20 smithi163 ceph-mon[25094]: from='client.24880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:47:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:20 smithi186 ceph-mon[25944]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:20 smithi186 ceph-mon[25944]: from='client.24880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:47:22.500 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T02:47:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T02:47:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-18T02:47:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:47:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T02:47:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T02:47:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:47:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T02:47:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-18T02:47:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:47:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "mds": {}, 2024-06-18T02:47:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T02:47:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T02:47:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:47:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T02:47:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-06-18T02:47:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T02:47:22.502 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T02:47:23.102 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:22 smithi163 ceph-mon[25094]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:23.102 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:22 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1641714381' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T02:47:23.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:22 smithi163 ceph-mon[29791]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:23.102 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:22 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1641714381' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T02:47:23.125 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph -s' 2024-06-18T02:47:23.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:22 smithi186 ceph-mon[25944]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:23.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:22 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1641714381' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T02:47:23.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:47:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:47:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:47:25.125 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:24 smithi163 ceph-mon[29791]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:25.126 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:24 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:25.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:24 smithi163 ceph-mon[25094]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:25.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:24 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:25.180 INFO:teuthology.orchestra.run.smithi163.stdout: cluster: 2024-06-18T02:47:25.180 INFO:teuthology.orchestra.run.smithi163.stdout: id: 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:47:25.180 INFO:teuthology.orchestra.run.smithi163.stdout: health: HEALTH_OK 2024-06-18T02:47:25.180 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:47:25.180 INFO:teuthology.orchestra.run.smithi163.stdout: services: 2024-06-18T02:47:25.180 INFO:teuthology.orchestra.run.smithi163.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-06-18T02:47:25.180 INFO:teuthology.orchestra.run.smithi163.stdout: mgr: y(active, since 5m), standbys: x 2024-06-18T02:47:25.180 INFO:teuthology.orchestra.run.smithi163.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-06-18T02:47:25.180 INFO:teuthology.orchestra.run.smithi163.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-06-18T02:47:25.180 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:47:25.180 INFO:teuthology.orchestra.run.smithi163.stdout: data: 2024-06-18T02:47:25.180 INFO:teuthology.orchestra.run.smithi163.stdout: pools: 6 pools, 161 pgs 2024-06-18T02:47:25.180 INFO:teuthology.orchestra.run.smithi163.stdout: objects: 241 objects, 585 KiB 2024-06-18T02:47:25.181 INFO:teuthology.orchestra.run.smithi163.stdout: usage: 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:47:25.181 INFO:teuthology.orchestra.run.smithi163.stdout: pgs: 161 active+clean 2024-06-18T02:47:25.181 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:47:25.181 INFO:teuthology.orchestra.run.smithi163.stdout: io: 2024-06-18T02:47:25.181 INFO:teuthology.orchestra.run.smithi163.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-06-18T02:47:25.181 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:47:25.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:24 smithi186 ceph-mon[25944]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:25.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:24 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:25.890 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ls' 2024-06-18T02:47:26.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:25 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2816699198' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-18T02:47:26.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:25 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2816699198' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-18T02:47:26.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:25 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2816699198' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-18T02:47:27.264 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:26 smithi163 ceph-mon[25094]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:27.264 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:26 smithi163 ceph-mon[29791]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:27.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:26 smithi186 ceph-mon[25944]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:27.753 INFO:teuthology.orchestra.run.smithi163.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-06-18T02:47:27.753 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi163=a;count:1 2024-06-18T02:47:27.753 INFO:teuthology.orchestra.run.smithi163.stdout:grafana ?:3000 1/1 3m ago 5m smithi186=a;count:1 2024-06-18T02:47:27.753 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-06-18T02:47:27.754 INFO:teuthology.orchestra.run.smithi163.stdout:mgr 2/2 3m ago 8m smithi163=y;smithi186=x;count:2 2024-06-18T02:47:27.754 INFO:teuthology.orchestra.run.smithi163.stdout:mon 3/3 3m ago 9m smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi186:172.21.15.186=b;count:3 2024-06-18T02:47:27.754 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi163=a;smithi186=b;count:2 2024-06-18T02:47:27.754 INFO:teuthology.orchestra.run.smithi163.stdout:osd 8 3m ago - 2024-06-18T02:47:27.754 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus ?:9095 1/1 3m ago 5m smithi186=a;count:1 2024-06-18T02:47:27.754 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-06-18T02:47:28.309 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-06-18T02:47:28.906 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:47:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:47:28] "GET /metrics HTTP/1.1" 200 208018 "" "Prometheus/2.33.4" 2024-06-18T02:47:29.252 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:28 smithi163 ceph-mon[29791]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:29.252 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:28 smithi163 ceph-mon[29791]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:47:29.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:28 smithi163 ceph-mon[25094]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:29.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:28 smithi163 ceph-mon[25094]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:47:29.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:28 smithi186 ceph-mon[25944]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:29.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:28 smithi186 ceph-mon[25944]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:47:30.511 INFO:teuthology.orchestra.run.smithi163.stdout:Scheduled to redeploy mgr.x on host 'smithi186' 2024-06-18T02:47:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[25094]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3118492345' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-18T02:47:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:47:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:47:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:47:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:47:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:47:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:47:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T02:47:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:47:31.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[29791]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:31.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3118492345' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-18T02:47:31.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:47:31.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:47:31.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:47:31.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:47:31.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:47:31.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:47:31.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T02:47:31.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:31 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:47:31.106 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps --refresh' 2024-06-18T02:47:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:31 smithi186 ceph-mon[25944]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:31 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3118492345' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-18T02:47:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:31 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:47:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:31 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:47:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:31 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:47:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:31 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:47:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:31 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:47:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:31 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:47:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:31 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T02:47:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:31 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:47:32.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:32 smithi163 ceph-mon[29791]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:47:32.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:32 smithi163 ceph-mon[29791]: Schedule redeploy daemon mgr.x 2024-06-18T02:47:32.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:32 smithi163 ceph-mon[29791]: Deploying daemon mgr.x on smithi186 2024-06-18T02:47:32.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:32 smithi163 ceph-mon[25094]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:47:32.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:32 smithi163 ceph-mon[25094]: Schedule redeploy daemon mgr.x 2024-06-18T02:47:32.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:32 smithi163 ceph-mon[25094]: Deploying daemon mgr.x on smithi186 2024-06-18T02:47:32.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:32 smithi186 ceph-mon[25944]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:47:32.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:32 smithi186 ceph-mon[25944]: Schedule redeploy daemon mgr.x 2024-06-18T02:47:32.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:32 smithi186 ceph-mon[25944]: Deploying daemon mgr.x on smithi186 2024-06-18T02:47:33.071 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T02:47:33.071 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (3m) 3m ago 4m 15.4M - ba2b418f427c 4277853c7745 2024-06-18T02:47:33.071 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (3m) 3m ago 3m 44.0M - 8.3.5 dad864ee21e9 f134c5a8fb6d 2024-06-18T02:47:33.071 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 running (3m) 3m ago 3m 57.4M - 3.5 e1d6a67b021e c28d1b6f808f 2024-06-18T02:47:33.071 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e d7519f47d8b0 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:9283 running (10m) 3m ago 10m 453M - 17.2.0 e1d6a67b021e 256bee9cc14e 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (10m) 3m ago 10m 49.6M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (8m) 3m ago 8m 39.4M 2048M 17.2.0 e1d6a67b021e a3618f734a9f 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (9m) 3m ago 9m 38.8M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (4m) 3m ago 4m 16.5M - 1dbe0e931976 4ed4364c9a98 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (4m) 3m ago 4m 18.7M - 1dbe0e931976 48a19a288113 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (8m) 3m ago 8m 49.6M 4016M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (8m) 3m ago 8m 49.6M 4016M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (7m) 3m ago 7m 44.2M 4016M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (7m) 3m ago 7m 45.7M 4016M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (6m) 3m ago 6m 52.0M 4272M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (6m) 3m ago 6m 44.3M 4272M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (6m) 3m ago 6m 43.6M 4272M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (5m) 3m ago 5m 45.8M 4272M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T02:47:33.072 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (3m) 3m ago 5m 44.2M - 514e6a882f6e 782012f610d3 2024-06-18T02:47:33.073 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (3m) 3m ago 3m 72.2M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T02:47:33.073 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (3m) 3m ago 3m 72.3M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T02:47:33.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:33 smithi163 ceph-mon[25094]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:33.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:33 smithi163 ceph-mon[29791]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:33.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:33 smithi186 ceph-mon[25944]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:33.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:47:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:47:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:47:33.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'sleep 180' 2024-06-18T02:47:35.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:35 smithi163 ceph-mon[29791]: from='client.24916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:47:35.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:35 smithi163 ceph-mon[29791]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:35.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:35 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:35.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:35 smithi163 ceph-mon[25094]: from='client.24916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:47:35.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:35 smithi163 ceph-mon[25094]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:35.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:35 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:35.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:35 smithi186 ceph-mon[25944]: from='client.24916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:47:35.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:35 smithi186 ceph-mon[25944]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:35.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:35 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:37.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:37 smithi163 ceph-mon[29791]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:37.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:37 smithi163 ceph-mon[25094]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:37.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:37 smithi186 ceph-mon[25944]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:39.012 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:47:38 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:47:38] "GET /metrics HTTP/1.1" 200 208024 "" "Prometheus/2.33.4" 2024-06-18T02:47:39.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:39 smithi163 ceph-mon[25094]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:39.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:39 smithi163 ceph-mon[29791]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:39.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:39 smithi186 ceph-mon[25944]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:41.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:41 smithi163 ceph-mon[29791]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:41.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:41 smithi163 ceph-mon[25094]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:41.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:41 smithi186 ceph-mon[25944]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:43.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:43 smithi163 ceph-mon[29791]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:43.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:43 smithi163 ceph-mon[25094]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:43.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:43 smithi186 ceph-mon[25944]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:43.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:47:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:47:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:47:45.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:45 smithi163 ceph-mon[25094]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:45.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:45 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:45.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:45 smithi163 ceph-mon[29791]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:45.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:45 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:45.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:45 smithi186 ceph-mon[25944]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:45.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:45 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:47.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:47 smithi163 ceph-mon[29791]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:47.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:47 smithi163 ceph-mon[25094]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:47.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:47 smithi186 ceph-mon[25944]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:49.024 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:47:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:47:48] "GET /metrics HTTP/1.1" 200 208013 "" "Prometheus/2.33.4" 2024-06-18T02:47:49.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:49 smithi163 ceph-mon[25094]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:49.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:49 smithi163 ceph-mon[29791]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:49.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:49 smithi186 ceph-mon[25944]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:51.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:51 smithi163 ceph-mon[25094]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:51.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:51 smithi163 ceph-mon[29791]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:51.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:51 smithi186 ceph-mon[25944]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:53.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:53 smithi163 ceph-mon[25094]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:53.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:53 smithi163 ceph-mon[29791]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:53.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:53 smithi186 ceph-mon[25944]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:53.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:47:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[27111]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:47:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:47:55.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:55 smithi163 ceph-mon[29791]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:55.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:55 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:55.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:55 smithi163 ceph-mon[25094]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:55.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:55 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:55.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:55 smithi186 ceph-mon[25944]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:55.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:55 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:47:57.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:57 smithi163 ceph-mon[25094]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:57.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:57 smithi163 ceph-mon[29791]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:57.317 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:57 smithi186 ceph-mon[25944]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:47:59.036 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:47:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:47:58] "GET /metrics HTTP/1.1" 200 208013 "" "Prometheus/2.33.4" 2024-06-18T02:47:59.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:47:59 smithi163 ceph-mon[29791]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:59.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:47:59 smithi163 ceph-mon[25094]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:47:59.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:47:59 smithi186 ceph-mon[25944]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:01.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:01 smithi163 ceph-mon[29791]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:01.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:01 smithi163 ceph-mon[25094]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:01.298 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:01 smithi186 ceph-mon[25944]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:01.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:01 smithi186 systemd[1]: Stopping Ceph mgr.x for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:48:01.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:01 smithi186 bash[45359]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr.x" found: no such container 2024-06-18T02:48:01.977 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:01 smithi186 podman[45370]: 2024-06-18 02:48:01.681259758 +0000 UTC m=+0.148653700 container died d7519f47d8b0573212e8da1d90f13e4d64c2ee1d5c50664d066b473ad56cb3ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, GIT_CLEAN=True, vcs-type=git, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, release=754, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vendor=Red Hat, Inc., ceph=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8) 2024-06-18T02:48:02.286 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 podman[45370]: 2024-06-18 02:48:02.077135271 +0000 UTC m=+0.544529209 container cleanup d7519f47d8b0573212e8da1d90f13e4d64c2ee1d5c50664d066b473ad56cb3ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, name=centos-stream, ceph=True, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, version=8, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vcs-type=git, io.k8s.display-name=CentOS Stream 8, release=754, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-06-18T02:48:02.286 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 bash[45370]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x 2024-06-18T02:48:02.287 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 bash[45397]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr.x" found: no such container 2024-06-18T02:48:02.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 podman[45384]: 2024-06-18 02:48:02.286943808 +0000 UTC m=+0.597623338 container remove d7519f47d8b0573212e8da1d90f13e4d64c2ee1d5c50664d066b473ad56cb3ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_CLEAN=True, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD) 2024-06-18T02:48:02.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-06-18T02:48:02.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-06-18T02:48:02.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 systemd[1]: Stopped Ceph mgr.x for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:48:02.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.x.service: Consumed 17.119s CPU time. 2024-06-18T02:48:02.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 systemd[1]: Starting Ceph mgr.x for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:48:02.966 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 podman[45496]: 2024-06-18 02:48:02.599298725 +0000 UTC m=+0.017635844 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T02:48:02.966 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 podman[45496]: 2024-06-18 02:48:02.696874078 +0000 UTC m=+0.115211240 container create d7bd16a633ffbf9fb48e1b7e707070457044f9d33f468e61f20ed46867637863 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, ceph=True) 2024-06-18T02:48:03.260 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:03 smithi186 ceph-mon[25944]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:03.261 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 podman[45496]: 2024-06-18 02:48:02.966256598 +0000 UTC m=+0.384593718 container init d7bd16a633ffbf9fb48e1b7e707070457044f9d33f468e61f20ed46867637863 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T02:48:03.261 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 podman[45496]: 2024-06-18 02:48:02.969487494 +0000 UTC m=+0.387824628 container start d7bd16a633ffbf9fb48e1b7e707070457044f9d33f468e61f20ed46867637863 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-3781e3a, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T02:48:03.261 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:02 smithi186 ceph-mgr[45515]: -- 172.21.15.186:0/3533344317 <== mon.2 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cbe7d4d0e0 con 0x55cbe7d2a800 2024-06-18T02:48:03.261 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:03 smithi186 bash[45496]: d7bd16a633ffbf9fb48e1b7e707070457044f9d33f468e61f20ed46867637863 2024-06-18T02:48:03.261 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:03 smithi186 systemd[1]: Started Ceph mgr.x for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:48:03.261 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:03.096+0000 7f95ecf3a040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T02:48:03.261 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:03.260+0000 7f95ecf3a040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:48:03.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:03 smithi163 ceph-mon[29791]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:03.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:03 smithi163 ceph-mon[25094]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:03.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:03.332+0000 7f95ecf3a040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T02:48:03.953 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:03.611+0000 7f95ecf3a040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T02:48:03.953 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:03.674+0000 7f95ecf3a040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T02:48:04.204 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:04 smithi186 ceph-mon[25944]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:04.204 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:04 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:04.204 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:04 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:04.205 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:04 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:48:04.205 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:04 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:48:04.205 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:04 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:04.205 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:04 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:04.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[29791]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:04.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:04.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:04.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:48:04.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:48:04.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:04.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[25094]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:48:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:48:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:04 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:04.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:04 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:04.711+0000 7f95ecf3a040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T02:48:05.153 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:04 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:04.838+0000 7f95ecf3a040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T02:48:05.153 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:04 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:04.908+0000 7f95ecf3a040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T02:48:05.431 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:05 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:05.153+0000 7f95ecf3a040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T02:48:05.431 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:05 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:05.217+0000 7f95ecf3a040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T02:48:05.431 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:05 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:05.288+0000 7f95ecf3a040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T02:48:05.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:05 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:05.454+0000 7f95ecf3a040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:48:06.234 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:05 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:05.903+0000 7f95ecf3a040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T02:48:06.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:06 smithi163 ceph-mon[25094]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:06.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:06 smithi163 ceph-mon[29791]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:06.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:06 smithi186 ceph-mon[25944]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:06.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:06.234+0000 7f95ecf3a040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T02:48:06.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:06.303+0000 7f95ecf3a040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T02:48:06.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:06.466+0000 7f95ecf3a040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-18T02:48:07.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:06.810+0000 7f95ecf3a040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T02:48:07.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:06.879+0000 7f95ecf3a040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T02:48:07.398 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:07.077+0000 7f95ecf3a040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T02:48:07.399 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:07.240+0000 7f95ecf3a040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T02:48:07.399 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:07.307+0000 7f95ecf3a040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T02:48:07.399 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:07.398+0000 7f95ecf3a040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T02:48:07.661 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:07.460+0000 7f95ecf3a040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T02:48:08.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:07 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:08.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:07 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:08.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:07 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:08.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:07 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:08.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:07 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:08.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:07 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:08.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:07 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:08.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:07 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:08.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:07 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:48:08.278 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:08.008+0000 7f95ecf3a040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T02:48:08.278 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:08.171+0000 7f95ecf3a040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T02:48:08.278 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:48:08] ENGINE Bus STARTING 2024-06-18T02:48:08.278 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: CherryPy Checker: 2024-06-18T02:48:08.278 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: The Application mounted at '' has an empty config. 2024-06-18T02:48:08.279 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:48:08.279 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:48:08] ENGINE Serving on http://:::9283 2024-06-18T02:48:08.279 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:48:08] ENGINE Bus STARTED 2024-06-18T02:48:09.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[25094]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:09.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[25094]: Standby manager daemon x restarted 2024-06-18T02:48:09.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[25094]: Standby manager daemon x started 2024-06-18T02:48:09.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T02:48:09.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:48:09.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T02:48:09.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:48:09.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:48:08] "GET /metrics HTTP/1.1" 200 208034 "" "Prometheus/2.33.4" 2024-06-18T02:48:09.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[29791]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:09.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[29791]: Standby manager daemon x restarted 2024-06-18T02:48:09.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[29791]: Standby manager daemon x started 2024-06-18T02:48:09.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T02:48:09.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:48:09.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T02:48:09.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:08 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:48:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-mon[25944]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-mon[25944]: Standby manager daemon x restarted 2024-06-18T02:48:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-mon[25944]: Standby manager daemon x started 2024-06-18T02:48:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T02:48:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:48:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T02:48:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:08 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:48:10.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:09 smithi163 ceph-mon[29791]: mgrmap e21: y(active, since 5m), standbys: x 2024-06-18T02:48:10.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:09 smithi163 ceph-mon[25094]: mgrmap e21: y(active, since 5m), standbys: x 2024-06-18T02:48:10.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:09 smithi186 ceph-mon[25944]: mgrmap e21: y(active, since 5m), standbys: x 2024-06-18T02:48:11.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:10 smithi163 ceph-mon[29791]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:11.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:10 smithi163 ceph-mon[25094]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:11.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:10 smithi186 ceph-mon[25944]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:13.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:12 smithi163 ceph-mon[29791]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:13.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:12 smithi163 ceph-mon[25094]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:13.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:12 smithi186 ceph-mon[25944]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:13.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:48:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:48:13.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:48:13.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:48:13.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:48:13.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:48:15.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:14 smithi163 ceph-mon[29791]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:15.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:14 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:15.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:14 smithi163 ceph-mon[25094]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:15.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:14 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:15.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:14 smithi186 ceph-mon[25944]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:14 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:48:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:48:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:48:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:48:16.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:48:16.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:48:16.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:48:16.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:48:16.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:48:16.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:48:16.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:48:16.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:48:17.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:16 smithi163 ceph-mon[29791]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:17.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:16 smithi163 ceph-mon[25094]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:17.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:16 smithi186 ceph-mon[25944]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:19.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:18 smithi163 ceph-mon[29791]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:19.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:18 smithi163 ceph-mon[25094]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:19.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:48:18 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:48:18] "GET /metrics HTTP/1.1" 200 208044 "" "Prometheus/2.33.4" 2024-06-18T02:48:19.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:18 smithi186 ceph-mon[25944]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:21.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:20 smithi163 ceph-mon[29791]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:21.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:20 smithi163 ceph-mon[25094]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:21.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:20 smithi186 ceph-mon[25944]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:23.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:22 smithi163 ceph-mon[25094]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:22 smithi163 ceph-mon[29791]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:23.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:22 smithi186 ceph-mon[25944]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:23.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:48:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:48:23.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:48:23.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:48:23.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:48:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:48:23.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:48:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:48:24.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:24 smithi163 ceph-mon[29791]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:24.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:24 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:24.967 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:24 smithi163 ceph-mon[25094]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:24.967 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:24 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:25.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:24 smithi186 ceph-mon[25944]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:25.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:24 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:27.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:26 smithi163 ceph-mon[25094]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:27.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:26 smithi163 ceph-mon[29791]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:27.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:26 smithi186 ceph-mon[25944]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:29.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:28 smithi163 ceph-mon[25094]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:29.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:28 smithi163 ceph-mon[29791]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:29.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:48:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:48:28] "GET /metrics HTTP/1.1" 200 208044 "" "Prometheus/2.33.4" 2024-06-18T02:48:29.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:28 smithi186 ceph-mon[25944]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:31.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:30 smithi163 ceph-mon[25094]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:31.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:30 smithi163 ceph-mon[29791]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:31.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:30 smithi186 ceph-mon[25944]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:33.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:32 smithi163 ceph-mon[29791]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:33.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:32 smithi163 ceph-mon[25094]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:33.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:32 smithi186 ceph-mon[25944]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:33.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:48:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:48:33.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:48:33.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:48:33.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:48:33.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:48:33.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:48:33.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:48:35.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:34 smithi163 ceph-mon[29791]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:35.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:34 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:35.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:34 smithi163 ceph-mon[25094]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:35.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:34 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:35.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:34 smithi186 ceph-mon[25944]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:35.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:34 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:37.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:36 smithi163 ceph-mon[29791]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:37.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:36 smithi163 ceph-mon[25094]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:37.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:36 smithi186 ceph-mon[25944]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:39.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:38 smithi163 ceph-mon[25094]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:39.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:38 smithi163 ceph-mon[29791]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:39.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:48:38 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:48:38] "GET /metrics HTTP/1.1" 200 208039 "" "Prometheus/2.33.4" 2024-06-18T02:48:39.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:38 smithi186 ceph-mon[25944]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:41.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:40 smithi163 ceph-mon[25094]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:41.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:40 smithi163 ceph-mon[29791]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:41.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:40 smithi186 ceph-mon[25944]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:43.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:42 smithi163 ceph-mon[29791]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:42 smithi163 ceph-mon[25094]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:43.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:42 smithi186 ceph-mon[25944]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:43.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:48:43.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:48:43.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:48:43.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:48:43.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:48:43.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:48:43.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:48:45.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:44 smithi163 ceph-mon[29791]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:45.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:44 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:44 smithi163 ceph-mon[25094]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:44 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:44 smithi186 ceph-mon[25944]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:44 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:47.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:46 smithi163 ceph-mon[29791]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:47.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:46 smithi163 ceph-mon[25094]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:47.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:46 smithi186 ceph-mon[25944]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:49.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:48 smithi163 ceph-mon[29791]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:49.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:48 smithi163 ceph-mon[25094]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:49.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:48:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:48:48] "GET /metrics HTTP/1.1" 200 208046 "" "Prometheus/2.33.4" 2024-06-18T02:48:49.067 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:48 smithi186 ceph-mon[25944]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:51.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:50 smithi163 ceph-mon[29791]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:51.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:50 smithi163 ceph-mon[25094]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:51.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:50 smithi186 ceph-mon[25944]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:52 smithi163 ceph-mon[29791]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:53.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:52 smithi163 ceph-mon[25094]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:52 smithi186 ceph-mon[25944]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:53.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:48:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:48:53.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:48:53.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:48:53.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:48:53.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:48:53.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:48:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:48:53.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:48:55.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:54 smithi163 ceph-mon[29791]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:55.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:54 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:55.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:54 smithi163 ceph-mon[25094]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:55.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:54 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:55.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:54 smithi186 ceph-mon[25944]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:55.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:54 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:48:57.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:56 smithi163 ceph-mon[29791]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:57.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:56 smithi163 ceph-mon[25094]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:57.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:56 smithi186 ceph-mon[25944]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:48:59.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:48:58 smithi163 ceph-mon[29791]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:59.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:48:58 smithi163 ceph-mon[25094]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:48:59.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:48:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:48:58] "GET /metrics HTTP/1.1" 200 208046 "" "Prometheus/2.33.4" 2024-06-18T02:48:59.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:48:58 smithi186 ceph-mon[25944]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:01.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:00 smithi163 ceph-mon[29791]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:01.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:00 smithi163 ceph-mon[25094]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:00 smithi186 ceph-mon[25944]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:03.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:02 smithi163 ceph-mon[29791]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:03.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:02 smithi163 ceph-mon[25094]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:02 smithi186 ceph-mon[25944]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:03.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:49:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:49:03.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:49:03.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:49:03.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:49:03.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:49:03.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:49:03.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:49:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:04 smithi163 ceph-mon[25094]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:04 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:05.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:04 smithi163 ceph-mon[29791]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:05.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:04 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:04 smithi186 ceph-mon[25944]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:04 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:07.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:06 smithi163 ceph-mon[25094]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:07.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:06 smithi163 ceph-mon[29791]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:07.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:06 smithi186 ceph-mon[25944]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:08.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:07 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:49:08.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:07 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:49:08.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:07 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:49:08.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:07 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:49:08.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:07 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:49:08.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:07 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:49:08.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:07 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:49:08.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:07 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:49:08.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:07 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:49:09.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:08 smithi163 ceph-mon[29791]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:09.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:08 smithi163 ceph-mon[25094]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:09.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:49:08 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:49:08] "GET /metrics HTTP/1.1" 200 208038 "" "Prometheus/2.33.4" 2024-06-18T02:49:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:08 smithi186 ceph-mon[25944]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:11.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:10 smithi186 ceph-mon[25944]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:11.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:10 smithi163 ceph-mon[25094]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:11.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:10 smithi163 ceph-mon[29791]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:12.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:12 smithi163 ceph-mon[29791]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:12.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:12 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:49:12.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:12 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:49:12.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:12 smithi163 ceph-mon[25094]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:12.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:12 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:49:12.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:12 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:49:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:12 smithi186 ceph-mon[25944]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:12 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:49:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:12 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:49:13.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:49:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:49:13.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:49:13.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:49:13.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:49:13.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:49:13.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:49:13.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:49:14.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:14 smithi186 ceph-mon[25944]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:14.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:14 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:14 smithi163 ceph-mon[29791]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:14 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:14 smithi163 ceph-mon[25094]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:14 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:15.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:49:15.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:49:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:49:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:49:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:49:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:49:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:49:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:49:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:49:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:49:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:49:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:49:16.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:16 smithi163 ceph-mon[29791]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:16 smithi163 ceph-mon[25094]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:16 smithi186 ceph-mon[25944]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:18.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:18 smithi163 ceph-mon[29791]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:18.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:18 smithi163 ceph-mon[25094]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:18.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:49:18 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:49:18] "GET /metrics HTTP/1.1" 200 208050 "" "Prometheus/2.33.4" 2024-06-18T02:49:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:18 smithi186 ceph-mon[25944]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:20.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:20 smithi163 ceph-mon[29791]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:20.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:20 smithi163 ceph-mon[25094]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:20.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:20 smithi186 ceph-mon[25944]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:22.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:22 smithi163 ceph-mon[25094]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:22.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:22 smithi163 ceph-mon[29791]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:22.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:22 smithi186 ceph-mon[25944]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:23.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:49:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:49:23.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:49:23.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:49:23.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:49:23.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:49:23.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:49:23.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:49:24.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:24 smithi163 ceph-mon[25094]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:24 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:24 smithi163 ceph-mon[29791]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:24 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:24 smithi186 ceph-mon[25944]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:24 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:26.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:26 smithi163 ceph-mon[29791]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:26 smithi163 ceph-mon[25094]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:26 smithi186 ceph-mon[25944]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:28.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:28 smithi163 ceph-mon[29791]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:28.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:28 smithi163 ceph-mon[25094]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:28.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:49:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:49:28] "GET /metrics HTTP/1.1" 200 208050 "" "Prometheus/2.33.4" 2024-06-18T02:49:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:28 smithi186 ceph-mon[25944]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:30.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:30 smithi163 ceph-mon[29791]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:30 smithi163 ceph-mon[25094]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:30 smithi186 ceph-mon[25944]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:32.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:32 smithi163 ceph-mon[29791]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:32.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:32 smithi163 ceph-mon[25094]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:32.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:32 smithi186 ceph-mon[25944]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:33.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:49:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:49:33.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:49:33.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:49:33.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:49:33.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:49:33.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:49:33.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:49:34.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:34 smithi186 ceph-mon[25944]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:34.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:34 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:34.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:34 smithi163 ceph-mon[29791]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:34.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:34 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:34 smithi163 ceph-mon[25094]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:34 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:36.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:36 smithi163 ceph-mon[29791]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:36 smithi163 ceph-mon[25094]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:36 smithi186 ceph-mon[25944]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:38.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:38 smithi163 ceph-mon[25094]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:38 smithi163 ceph-mon[29791]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:38.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:49:38 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:49:38] "GET /metrics HTTP/1.1" 200 208049 "" "Prometheus/2.33.4" 2024-06-18T02:49:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:38 smithi186 ceph-mon[25944]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:40.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:40 smithi163 ceph-mon[29791]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:40.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:40 smithi163 ceph-mon[25094]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:40 smithi186 ceph-mon[25944]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:42.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:42 smithi163 ceph-mon[25094]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:42.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:42 smithi163 ceph-mon[29791]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:42 smithi186 ceph-mon[25944]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:43.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:49:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:49:43.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:49:43.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:49:43.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:49:43.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:49:43.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:49:43.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:49:44.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:44 smithi186 ceph-mon[25944]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:44.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:44 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:44.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:44 smithi163 ceph-mon[29791]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:44.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:44 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:44 smithi163 ceph-mon[25094]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:44 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:46.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:46 smithi163 ceph-mon[29791]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:46.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:46 smithi163 ceph-mon[25094]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:46 smithi186 ceph-mon[25944]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:48.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:48 smithi163 ceph-mon[29791]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:48.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:49:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:49:48] "GET /metrics HTTP/1.1" 200 208047 "" "Prometheus/2.33.4" 2024-06-18T02:49:48.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:48 smithi163 ceph-mon[25094]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:48 smithi186 ceph-mon[25944]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:50.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:50 smithi163 ceph-mon[29791]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:50.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:50 smithi163 ceph-mon[25094]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:50.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:50 smithi186 ceph-mon[25944]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:52.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:52 smithi163 ceph-mon[29791]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:52.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:52 smithi163 ceph-mon[25094]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:52.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:52 smithi186 ceph-mon[25944]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:53.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:49:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:49:53.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:49:53.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:49:53.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:49:53.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:49:53.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:49:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:49:53.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:49:54.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:54 smithi163 ceph-mon[25094]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:54.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:54 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:54.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:54 smithi163 ceph-mon[29791]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:54.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:54 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:54.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:54 smithi186 ceph-mon[25944]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:54 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:49:56.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:56 smithi163 ceph-mon[25094]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:56.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:56 smithi163 ceph-mon[29791]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:56 smithi186 ceph-mon[25944]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:49:58.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:49:58 smithi163 ceph-mon[29791]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:58.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:49:58 smithi163 ceph-mon[25094]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:49:58.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:49:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:49:58] "GET /metrics HTTP/1.1" 200 208047 "" "Prometheus/2.33.4" 2024-06-18T02:49:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:49:58 smithi186 ceph-mon[25944]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:00.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:00 smithi163 ceph-mon[29791]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:00.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:00 smithi163 ceph-mon[29791]: overall HEALTH_OK 2024-06-18T02:50:00.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:00 smithi163 ceph-mon[25094]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:00.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:00 smithi163 ceph-mon[25094]: overall HEALTH_OK 2024-06-18T02:50:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:00 smithi186 ceph-mon[25944]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:00 smithi186 ceph-mon[25944]: overall HEALTH_OK 2024-06-18T02:50:02.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:02 smithi163 ceph-mon[29791]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:02 smithi163 ceph-mon[25094]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:02 smithi186 ceph-mon[25944]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:03.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:50:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:50:03.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:50:03.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:50:03.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:50:03.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:50:03.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:50:03.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:50:04.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:04 smithi163 ceph-mon[25094]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:04.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:04 smithi163 ceph-mon[29791]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:04.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:04 smithi186 ceph-mon[25944]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:05.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:05 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:05.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:05 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:05.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:05 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:06.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:06 smithi163 ceph-mon[29791]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:06.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:06 smithi163 ceph-mon[25094]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:06 smithi186 ceph-mon[25944]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:08.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:08 smithi163 ceph-mon[29791]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:08.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:08 smithi163 ceph-mon[25094]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:08.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:08 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:50:08] "GET /metrics HTTP/1.1" 200 208045 "" "Prometheus/2.33.4" 2024-06-18T02:50:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:08 smithi186 ceph-mon[25944]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:10.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:10 smithi163 ceph-mon[29791]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:10.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:10 smithi163 ceph-mon[25094]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:10.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:10 smithi186 ceph-mon[25944]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:12.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:12 smithi163 ceph-mon[29791]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:12.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:12 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:50:12.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:12 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:50:12.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:12 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:50:12.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:12 smithi163 ceph-mon[25094]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:12.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:12 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:50:12.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:12 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:50:12.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:12 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:50:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:12 smithi186 ceph-mon[25944]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:12 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:50:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:12 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:50:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:12 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' 2024-06-18T02:50:13.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:50:13.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:50:13.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:50:13.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:50:13.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:50:13.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:50:13.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:50:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:14 smithi163 ceph-mon[29791]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:14 smithi163 ceph-mon[25094]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:14 smithi186 ceph-mon[25944]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:15.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:15 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:15.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:50:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:15 smithi163 ceph-mon[29791]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:50:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:50:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:15 smithi163 ceph-mon[29791]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:50:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:15 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:50:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:15 smithi163 ceph-mon[25094]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:50:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:50:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:15 smithi163 ceph-mon[25094]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:50:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:15 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:50:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:15 smithi186 ceph-mon[25944]: from='mgr.24343 172.21.15.163:0/3157019062' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:50:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:50:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:15 smithi186 ceph-mon[25944]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:50:16.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:16 smithi163 ceph-mon[29791]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:16 smithi163 ceph-mon[25094]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:16 smithi186 ceph-mon[25944]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:18.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:18 smithi163 ceph-mon[29791]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:18.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:18 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:50:18] "GET /metrics HTTP/1.1" 200 208053 "" "Prometheus/2.33.4" 2024-06-18T02:50:18.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:18 smithi163 ceph-mon[25094]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:18 smithi186 ceph-mon[25944]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:20.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:20 smithi163 ceph-mon[29791]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:20.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:20 smithi163 ceph-mon[25094]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:20.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:20 smithi186 ceph-mon[25944]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:22.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:22 smithi163 ceph-mon[29791]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:22.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:22 smithi163 ceph-mon[25094]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:22.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:22 smithi186 ceph-mon[25944]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:23.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:50:23.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:50:23.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:50:23.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:50:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:50:23.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:50:23.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:50:24.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:24 smithi163 ceph-mon[25094]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:24 smithi163 ceph-mon[29791]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:24 smithi186 ceph-mon[25944]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:25.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:25 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:25.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:25 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:25 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:26.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:26 smithi163 ceph-mon[29791]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:26 smithi163 ceph-mon[25094]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:26 smithi186 ceph-mon[25944]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:28 smithi186 ceph-mon[25944]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:29.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:28 smithi163 ceph-mon[29791]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:29.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:28 smithi163 ceph-mon[25094]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:29.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:50:28] "GET /metrics HTTP/1.1" 200 208053 "" "Prometheus/2.33.4" 2024-06-18T02:50:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:30 smithi186 ceph-mon[25944]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:31.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:30 smithi163 ceph-mon[29791]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:31.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:30 smithi163 ceph-mon[25094]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:32 smithi186 ceph-mon[25944]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:33.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:32 smithi163 ceph-mon[29791]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:33.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:32 smithi163 ceph-mon[25094]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:33.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:50:33.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:50:33.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:50:33.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:50:33.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:50:33.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:50:33.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:50:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:34 smithi186 ceph-mon[25944]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:35.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:34 smithi163 ceph-mon[29791]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:35.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:34 smithi163 ceph-mon[25094]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:35.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:35 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:35.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:35 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:35.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:35 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:35.849 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps' 2024-06-18T02:50:36.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:36 smithi163 ceph-mon[25094]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:36 smithi163 ceph-mon[29791]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:36 smithi186 ceph-mon[25944]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:37.780 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T02:50:37.780 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (6m) 2m ago 7m 23.4M - ba2b418f427c 4277853c7745 2024-06-18T02:50:37.780 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (6m) 86s ago 6m 53.9M - 8.3.5 dad864ee21e9 f134c5a8fb6d 2024-06-18T02:50:37.781 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 running (6m) 2m ago 6m 51.1M - 3.5 e1d6a67b021e c28d1b6f808f 2024-06-18T02:50:37.781 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283 running (2m) 86s ago 11m 438M - 19.0.0-3124-g3781e3a5 af93c036e016 d7bd16a633ff 2024-06-18T02:50:37.781 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:9283 running (13m) 2m ago 13m 461M - 17.2.0 e1d6a67b021e 256bee9cc14e 2024-06-18T02:50:37.781 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (13m) 2m ago 13m 54.0M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T02:50:37.781 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (12m) 86s ago 12m 41.1M 2048M 17.2.0 e1d6a67b021e a3618f734a9f 2024-06-18T02:50:37.781 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (12m) 2m ago 12m 40.4M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T02:50:37.781 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (8m) 2m ago 8m 17.3M - 1dbe0e931976 4ed4364c9a98 2024-06-18T02:50:37.781 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (7m) 86s ago 7m 17.2M - 1dbe0e931976 48a19a288113 2024-06-18T02:50:37.781 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (11m) 2m ago 11m 52.0M 4016M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T02:50:37.781 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (11m) 2m ago 11m 51.3M 4016M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T02:50:37.781 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (10m) 2m ago 10m 46.8M 4016M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T02:50:37.781 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (10m) 2m ago 10m 49.2M 4016M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T02:50:37.781 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (9m) 86s ago 9m 55.5M 4272M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T02:50:37.782 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (9m) 86s ago 9m 48.5M 4272M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T02:50:37.782 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (9m) 86s ago 9m 46.7M 4272M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T02:50:37.782 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (8m) 86s ago 8m 49.3M 4272M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T02:50:37.782 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (6m) 86s ago 8m 66.7M - 514e6a882f6e 782012f610d3 2024-06-18T02:50:37.782 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (6m) 2m ago 6m 73.5M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T02:50:37.782 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (6m) 86s ago 6m 74.0M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T02:50:38.445 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions' 2024-06-18T02:50:38.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:38 smithi163 ceph-mon[29791]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:38.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:38 smithi163 ceph-mon[29791]: from='client.15021 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:50:38.694 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:38 smithi163 ceph-mon[25094]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:38.694 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:38 smithi163 ceph-mon[25094]: from='client.15021 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:50:38.694 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:38 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:50:38] "GET /metrics HTTP/1.1" 200 208049 "" "Prometheus/2.33.4" 2024-06-18T02:50:39.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:38 smithi186 ceph-mon[25944]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:39.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:38 smithi186 ceph-mon[25944]: from='client.15021 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:50:40.452 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T02:50:40.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T02:50:40.452 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-18T02:50:40.452 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:50:40.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T02:50:40.452 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-06-18T02:50:40.452 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 1 2024-06-18T02:50:40.452 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:50:40.452 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T02:50:40.452 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-18T02:50:40.452 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:50:40.452 INFO:teuthology.orchestra.run.smithi163.stdout: "mds": {}, 2024-06-18T02:50:40.453 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T02:50:40.453 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T02:50:40.453 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:50:40.453 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T02:50:40.453 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-06-18T02:50:40.453 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 1 2024-06-18T02:50:40.453 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T02:50:40.453 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T02:50:40.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:40 smithi163 ceph-mon[25094]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:40.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:40 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3213535706' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T02:50:40.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:40 smithi163 ceph-mon[29791]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:40.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:40 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3213535706' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T02:50:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:40 smithi186 ceph-mon[25944]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:40 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3213535706' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T02:50:41.083 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph -s' 2024-06-18T02:50:42.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:42 smithi163 ceph-mon[29791]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:42.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:42 smithi163 ceph-mon[25094]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:42.894 INFO:teuthology.orchestra.run.smithi163.stdout: cluster: 2024-06-18T02:50:42.894 INFO:teuthology.orchestra.run.smithi163.stdout: id: 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:50:42.895 INFO:teuthology.orchestra.run.smithi163.stdout: health: HEALTH_OK 2024-06-18T02:50:42.895 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:50:42.895 INFO:teuthology.orchestra.run.smithi163.stdout: services: 2024-06-18T02:50:42.895 INFO:teuthology.orchestra.run.smithi163.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-06-18T02:50:42.895 INFO:teuthology.orchestra.run.smithi163.stdout: mgr: y(active, since 8m), standbys: x 2024-06-18T02:50:42.895 INFO:teuthology.orchestra.run.smithi163.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-06-18T02:50:42.895 INFO:teuthology.orchestra.run.smithi163.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-06-18T02:50:42.895 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:50:42.895 INFO:teuthology.orchestra.run.smithi163.stdout: data: 2024-06-18T02:50:42.895 INFO:teuthology.orchestra.run.smithi163.stdout: pools: 6 pools, 161 pgs 2024-06-18T02:50:42.895 INFO:teuthology.orchestra.run.smithi163.stdout: objects: 241 objects, 585 KiB 2024-06-18T02:50:42.895 INFO:teuthology.orchestra.run.smithi163.stdout: usage: 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:50:42.895 INFO:teuthology.orchestra.run.smithi163.stdout: pgs: 161 active+clean 2024-06-18T02:50:42.896 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:50:42.896 INFO:teuthology.orchestra.run.smithi163.stdout: io: 2024-06-18T02:50:42.896 INFO:teuthology.orchestra.run.smithi163.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-06-18T02:50:42.896 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:50:43.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:42 smithi186 ceph-mon[25944]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:43.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph health detail' 2024-06-18T02:50:43.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:50:43.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:43 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/4145030538' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-18T02:50:43.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:43 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/4145030538' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-18T02:50:43.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:50:43.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:50:43.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:50:43.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:50:43.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:50:43.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:50:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:43 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/4145030538' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-18T02:50:44.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:44 smithi163 ceph-mon[29791]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:44.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:44 smithi163 ceph-mon[25094]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:44 smithi186 ceph-mon[25944]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:45.482 INFO:teuthology.orchestra.run.smithi163.stdout:HEALTH_OK 2024-06-18T02:50:45.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:45 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:45.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:45 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3371786060' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T02:50:45.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:45 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:45.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:45 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3371786060' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T02:50:46.050 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-06-18T02:50:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:45 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:45 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3371786060' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T02:50:46.688 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:46 smithi163 ceph-mon[25094]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:46.688 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:46 smithi163 ceph-mon[29791]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:47.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:46 smithi186 ceph-mon[25944]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:48.042 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T02:50:48.623 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph mgr fail' 2024-06-18T02:50:48.931 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:48 smithi163 ceph-mon[29791]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:48.931 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:48 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2209512244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T02:50:48.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:48 smithi163 ceph-mon[25094]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:48.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:48 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2209512244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T02:50:48.931 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:48 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:50:48] "GET /metrics HTTP/1.1" 200 208040 "" "Prometheus/2.33.4" 2024-06-18T02:50:49.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:48 smithi186 ceph-mon[25944]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:50:49.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:48 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2209512244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T02:50:51.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:50 smithi163 ceph-mon[25094]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:51.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:50 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2185496613' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T02:50:51.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:50 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T02:50:51.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:50 smithi163 ceph-mon[25094]: osdmap e95: 8 total, 8 up, 8 in 2024-06-18T02:50:51.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:50 smithi163 ceph-mon[29791]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:51.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:50 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2185496613' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T02:50:51.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:50 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T02:50:51.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:50 smithi163 ceph-mon[29791]: osdmap e95: 8 total, 8 up, 8 in 2024-06-18T02:50:51.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:50 smithi186 ceph-mon[25944]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:50:51.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:50 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2185496613' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T02:50:51.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:50 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T02:50:51.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:50 smithi186 ceph-mon[25944]: osdmap e95: 8 total, 8 up, 8 in 2024-06-18T02:50:51.695 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ignoring --setuser ceph since I am not root 2024-06-18T02:50:51.695 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ignoring --setgroup ceph since I am not root 2024-06-18T02:50:51.695 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mgr[25326]: -- 172.21.15.163:0/896145274 <== mon.1 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56550e4b4340 con 0x56550e48c800 2024-06-18T02:50:51.695 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:51.568+0000 7fdc2988e000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T02:50:51.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: Standby manager daemon y started 2024-06-18T02:50:51.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:50:51.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:50:51.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:50:51.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-18T02:50:51.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: mgrmap e22: x(active, starting, since 0.870201s), standbys: y 2024-06-18T02:50:51.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T02:50:51.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T02:50:51.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:50:51.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:50:51.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:50:51.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:50:51.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:50:51.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:50:51.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:50:51.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:50:51.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T02:50:51.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T02:50:51.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T02:50:51.697 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: Standby manager daemon y started 2024-06-18T02:50:51.697 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:50:51.697 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: mgrmap e22: x(active, starting, since 0.870201s), standbys: y 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:50:51.698 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T02:50:51.699 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T02:50:51.699 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T02:50:51.707 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:50:51] ENGINE Bus STOPPING 2024-06-18T02:50:51.707 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: Standby manager daemon y started 2024-06-18T02:50:51.707 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:50:51.707 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:50:51.707 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:50:51.707 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-18T02:50:51.707 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: mgrmap e22: x(active, starting, since 0.870201s), standbys: y 2024-06-18T02:50:51.708 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T02:50:51.708 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T02:50:51.708 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:50:51.708 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:50:51.708 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:50:51.708 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:50:51.708 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:50:51.708 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:50:51.708 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:50:51.708 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:50:51.708 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T02:50:51.708 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T02:50:51.708 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T02:50:52.004 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:51.702+0000 7fdc2988e000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:50:52.004 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:51 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:51.804+0000 7fdc2988e000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T02:50:52.031 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:50:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-18T02:50:52.031 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:50:51] ENGINE Bus STOPPED 2024-06-18T02:50:52.031 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:50:51] ENGINE Bus STARTING 2024-06-18T02:50:52.031 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:50:51] ENGINE Serving on http://:::9283 2024-06-18T02:50:52.031 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:50:51] ENGINE Bus STARTED 2024-06-18T02:50:52.035 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'sleep 180' 2024-06-18T02:50:52.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:52.135+0000 7fdc2988e000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T02:50:52.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:52.228+0000 7fdc2988e000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T02:50:52.605 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[25094]: Manager daemon x is now available 2024-06-18T02:50:52.605 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:52.605 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[25094]: Queued rgw.foo for migration 2024-06-18T02:50:52.605 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[25094]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-06-18T02:50:52.605 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:52.605 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:50:52.605 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:50:52.606 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:50:52.606 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-18T02:50:52.606 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-18T02:50:52.606 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:52.606 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[29791]: Manager daemon x is now available 2024-06-18T02:50:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[29791]: Queued rgw.foo for migration 2024-06-18T02:50:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[29791]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-06-18T02:50:53.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:53.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:50:53.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:50:53.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:50:53.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-18T02:50:53.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-18T02:50:53.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:53.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:52 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:52 smithi186 ceph-mon[25944]: Manager daemon x is now available 2024-06-18T02:50:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:52 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:52 smithi186 ceph-mon[25944]: Queued rgw.foo for migration 2024-06-18T02:50:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:52 smithi186 ceph-mon[25944]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-06-18T02:50:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:52 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:52 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:50:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:52 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:50:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:52 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:50:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:52 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-18T02:50:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:52 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-18T02:50:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:52 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:53.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:52 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:53.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:50:53] "GET /metrics HTTP/1.1" 200 35103 "" "Prometheus/2.33.4" 2024-06-18T02:50:53.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:50:53.264+0000 7f95449e8640 -1 log_channel(cephadm) log [ERR] : [18/Jun/2024:02:50:53] ENGINE Error in HTTPServer.serve 2024-06-18T02:50:53.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T02:50:53.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T02:50:53.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self._connections.run(self.expiration_interval) 2024-06-18T02:50:53.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T02:50:53.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self._run(expiration_interval) 2024-06-18T02:50:53.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T02:50:53.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T02:50:53.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T02:50:53.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T02:50:53.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T02:50:53.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: s = self.context.wrap_socket( 2024-06-18T02:50:53.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T02:50:53.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.sslsocket_class._create( 2024-06-18T02:50:53.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T02:50:53.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self.do_handshake() 2024-06-18T02:50:53.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T02:50:53.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self._sslobj.do_handshake() 2024-06-18T02:50:53.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T02:50:53.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:50:53.621 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:53.412+0000 7fdc2988e000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T02:50:53.621 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:53.582+0000 7fdc2988e000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T02:50:53.621 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:50:53.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:50:53.622 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:50:53.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T02:50:53.622 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:50:53.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:50:53.948 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-mon[25944]: Deploying cephadm binary to smithi186 2024-06-18T02:50:53.948 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-mon[25944]: Deploying cephadm binary to smithi163 2024-06-18T02:50:53.948 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-mon[25944]: mgrmap e23: x(active, since 2s), standbys: y 2024-06-18T02:50:53.948 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-mon[25944]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:50:53.948 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:53 smithi186 ceph-mon[25944]: [18/Jun/2024:02:50:52] ENGINE Bus STARTING 2024-06-18T02:50:54.001 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:53.674+0000 7fdc2988e000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T02:50:54.001 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-mon[29791]: Deploying cephadm binary to smithi186 2024-06-18T02:50:54.001 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-mon[29791]: Deploying cephadm binary to smithi163 2024-06-18T02:50:54.001 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-mon[29791]: mgrmap e23: x(active, since 2s), standbys: y 2024-06-18T02:50:54.001 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-mon[29791]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:50:54.001 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-mon[29791]: [18/Jun/2024:02:50:52] ENGINE Bus STARTING 2024-06-18T02:50:54.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-mon[25094]: Deploying cephadm binary to smithi186 2024-06-18T02:50:54.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-mon[25094]: Deploying cephadm binary to smithi163 2024-06-18T02:50:54.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-mon[25094]: mgrmap e23: x(active, since 2s), standbys: y 2024-06-18T02:50:54.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-mon[25094]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:50:54.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:53 smithi163 ceph-mon[25094]: [18/Jun/2024:02:50:52] ENGINE Bus STARTING 2024-06-18T02:50:54.284 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:54.000+0000 7fdc2988e000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T02:50:54.284 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:54.101+0000 7fdc2988e000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T02:50:54.284 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:54.200+0000 7fdc2988e000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T02:50:54.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:54.491+0000 7fdc2988e000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:50:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:54 smithi186 ceph-mon[25944]: [18/Jun/2024:02:50:53] ENGINE Serving on http://172.21.15.186:8765 2024-06-18T02:50:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:54 smithi186 ceph-mon[25944]: [18/Jun/2024:02:50:53] ENGINE Serving on https://172.21.15.186:7150 2024-06-18T02:50:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:54 smithi186 ceph-mon[25944]: [18/Jun/2024:02:50:53] ENGINE Bus STARTED 2024-06-18T02:50:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:54 smithi186 ceph-mon[25944]: [18/Jun/2024:02:50:53] ENGINE Error in HTTPServer.serve 2024-06-18T02:50:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T02:50:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T02:50:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._connections.run(self.expiration_interval) 2024-06-18T02:50:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._run(expiration_interval) 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: s = self.context.wrap_socket( 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.sslsocket_class._create( 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: self.do_handshake() 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._sslobj.do_handshake() 2024-06-18T02:50:54.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T02:50:54.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:54 smithi186 ceph-mon[25944]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:50:54.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-mon[25094]: [18/Jun/2024:02:50:53] ENGINE Serving on http://172.21.15.186:8765 2024-06-18T02:50:54.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-mon[25094]: [18/Jun/2024:02:50:53] ENGINE Serving on https://172.21.15.186:7150 2024-06-18T02:50:54.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-mon[25094]: [18/Jun/2024:02:50:53] ENGINE Bus STARTED 2024-06-18T02:50:54.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-mon[25094]: [18/Jun/2024:02:50:53] ENGINE Error in HTTPServer.serve 2024-06-18T02:50:54.971 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:50:54.971 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T02:50:54.971 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._connections.run(self.expiration_interval) 2024-06-18T02:50:54.971 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T02:50:54.971 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._run(expiration_interval) 2024-06-18T02:50:54.971 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T02:50:54.971 INFO:journalctl@ceph.mon.a.smithi163.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T02:50:54.971 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T02:50:54.971 INFO:journalctl@ceph.mon.a.smithi163.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T02:50:54.971 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T02:50:54.971 INFO:journalctl@ceph.mon.a.smithi163.stdout: s = self.context.wrap_socket( 2024-06-18T02:50:54.972 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T02:50:54.972 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.sslsocket_class._create( 2024-06-18T02:50:54.972 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T02:50:54.972 INFO:journalctl@ceph.mon.a.smithi163.stdout: self.do_handshake() 2024-06-18T02:50:54.972 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T02:50:54.972 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._sslobj.do_handshake() 2024-06-18T02:50:54.972 INFO:journalctl@ceph.mon.a.smithi163.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T02:50:54.972 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-mon[25094]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:50:54.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-mon[29791]: [18/Jun/2024:02:50:53] ENGINE Serving on http://172.21.15.186:8765 2024-06-18T02:50:54.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-mon[29791]: [18/Jun/2024:02:50:53] ENGINE Serving on https://172.21.15.186:7150 2024-06-18T02:50:54.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-mon[29791]: [18/Jun/2024:02:50:53] ENGINE Bus STARTED 2024-06-18T02:50:54.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-mon[29791]: [18/Jun/2024:02:50:53] ENGINE Error in HTTPServer.serve 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._connections.run(self.expiration_interval) 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._run(expiration_interval) 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: s = self.context.wrap_socket( 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.sslsocket_class._create( 2024-06-18T02:50:54.973 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T02:50:54.974 INFO:journalctl@ceph.mon.c.smithi163.stdout: self.do_handshake() 2024-06-18T02:50:54.974 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T02:50:54.974 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._sslobj.do_handshake() 2024-06-18T02:50:54.974 INFO:journalctl@ceph.mon.c.smithi163.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T02:50:54.974 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-mon[29791]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:50:55.273 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:54 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:54.970+0000 7fdc2988e000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T02:50:55.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:55 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:55.522+0000 7fdc2988e000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T02:50:55.924 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:55 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:55.924 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:55 smithi163 ceph-mon[29791]: mgrmap e24: x(active, since 4s), standbys: y 2024-06-18T02:50:55.924 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:55 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:55.924 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:55 smithi163 ceph-mon[25094]: mgrmap e24: x(active, since 4s), standbys: y 2024-06-18T02:50:55.924 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:55 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:55.620+0000 7fdc2988e000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T02:50:55.925 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:55 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:55.923+0000 7fdc2988e000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T02:50:55.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:55 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:50:55.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:55 smithi186 ceph-mon[25944]: mgrmap e24: x(active, since 4s), standbys: y 2024-06-18T02:50:56.243 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:56.024+0000 7fdc2988e000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T02:50:56.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:56.456+0000 7fdc2988e000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T02:50:56.922 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:56 smithi163 ceph-mon[29791]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:50:56.923 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:56 smithi163 ceph-mon[25094]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:50:56.923 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:56.592+0000 7fdc2988e000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T02:50:56.923 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:56.691+0000 7fdc2988e000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T02:50:56.923 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:56.831+0000 7fdc2988e000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T02:50:56.923 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:56.922+0000 7fdc2988e000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T02:50:56.978 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:56 smithi186 ceph-mon[25944]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:50:57.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:57.503+0000 7fdc2988e000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T02:50:58.237 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:57.888+0000 7fdc2988e000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T02:50:58.237 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:50:57] ENGINE Bus STARTING 2024-06-18T02:50:58.237 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: CherryPy Checker: 2024-06-18T02:50:58.237 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: The Application mounted at '' has an empty config. 2024-06-18T02:50:58.237 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: 2024-06-18T02:50:58.238 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:50:57] ENGINE Serving on http://:::9283 2024-06-18T02:50:58.238 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: [18/Jun/2024:02:50:57] ENGINE Bus STARTED 2024-06-18T02:50:58.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:50:58.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: Adjusting osd_memory_target on smithi186 to 3248M 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: Standby manager daemon y restarted 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: Standby manager daemon y started 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.163:0/4114260474' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.163:0/4114260474' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:50:58.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.163:0/4114260474' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-18T02:50:58.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.163:0/4114260474' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:50:58.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: Adjusting osd_memory_target on smithi186 to 3248M 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: Standby manager daemon y restarted 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: Standby manager daemon y started 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.163:0/4114260474' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.163:0/4114260474' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.163:0/4114260474' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.163:0/4114260474' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:50:58.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:50:58.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:50:58.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:50:58.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:50:58.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: Adjusting osd_memory_target on smithi186 to 3248M 2024-06-18T02:50:58.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: Standby manager daemon y restarted 2024-06-18T02:50:58.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: Standby manager daemon y started 2024-06-18T02:50:58.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.163:0/4114260474' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.163:0/4114260474' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.163:0/4114260474' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.163:0/4114260474' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-18T02:50:58.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:50:58.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:50:58.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:58 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:50:59.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:50:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:50:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:50:59.511 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:50:59 smithi163 systemd[1]: Stopping Ceph node-exporter.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:50:59.512 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:50:59 smithi163 bash[58670]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter.a" found: no such container 2024-06-18T02:50:59.512 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:50:59.208Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:50:59.792 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:50:59 smithi163 podman[58681]: 2024-06-18 02:50:59.512075368 +0000 UTC m=+0.114482236 container died 4ed4364c9a98af4f1c183d062706b0767a85d75beccd2605be9b27bb00fa47f7 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:00.077 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:50:59 smithi163 podman[58681]: 2024-06-18 02:50:59.82650122 +0000 UTC m=+0.428908088 container cleanup 4ed4364c9a98af4f1c183d062706b0767a85d75beccd2605be9b27bb00fa47f7 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:00.077 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:50:59 smithi163 bash[58681]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a 2024-06-18T02:51:00.078 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 podman[58695]: 2024-06-18 02:51:00.027739478 +0000 UTC m=+0.518983387 container remove 4ed4364c9a98af4f1c183d062706b0767a85d75beccd2605be9b27bb00fa47f7 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:00.078 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-06-18T02:51:00.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: Adjusting osd_memory_target on smithi163 to 2992M 2024-06-18T02:51:00.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T02:51:00.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T02:51:00.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T02:51:00.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T02:51:00.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:51:00.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:51:00.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T02:51:00.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T02:51:00.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: mgrmap e25: x(active, since 8s), standbys: y 2024-06-18T02:51:00.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: Adjusting osd_memory_target on smithi163 to 2992M 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: mgrmap e25: x(active, since 8s), standbys: y 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:50:59 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: Adjusting osd_memory_target on smithi163 to 2992M 2024-06-18T02:51:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T02:51:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T02:51:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T02:51:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: mgrmap e25: x(active, since 8s), standbys: y 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:50:59 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:00.338 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 bash[58708]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter.a" found: no such container 2024-06-18T02:51:00.338 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-06-18T02:51:00.338 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 systemd[1]: Stopped Ceph node-exporter.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:51:00.338 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.a.service: Consumed 3.797s CPU time. 2024-06-18T02:51:00.338 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 systemd[1]: Starting Ceph node-exporter.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:51:00.338 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 podman[58771]: 2024-06-18 02:51:00.211322089 +0000 UTC m=+0.016902521 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-06-18T02:51:00.338 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 podman[58771]: 2024-06-18 02:51:00.338512076 +0000 UTC m=+0.144092539 container create 8d0aecfcef8a83349c1cdc0291918331c3c0ea8c24f66f604ee3599aa9e25f5f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:00.648 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 podman[58771]: 2024-06-18 02:51:00.556358128 +0000 UTC m=+0.361938568 container init 8d0aecfcef8a83349c1cdc0291918331c3c0ea8c24f66f604ee3599aa9e25f5f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:00.648 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 podman[58771]: 2024-06-18 02:51:00.559318211 +0000 UTC m=+0.364898641 container start 8d0aecfcef8a83349c1cdc0291918331c3c0ea8c24f66f604ee3599aa9e25f5f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:00.648 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.562Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-06-18T02:51:00.648 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.562Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-06-18T02:51:00.648 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-06-18T02:51:00.648 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-06-18T02:51:00.648 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-06-18T02:51:00.648 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=arp 2024-06-18T02:51:00.648 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=bcache 2024-06-18T02:51:00.648 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=bonding 2024-06-18T02:51:00.648 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=btrfs 2024-06-18T02:51:00.649 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=conntrack 2024-06-18T02:51:00.649 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=cpu 2024-06-18T02:51:00.649 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-06-18T02:51:00.649 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=diskstats 2024-06-18T02:51:00.649 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=dmi 2024-06-18T02:51:00.649 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=edac 2024-06-18T02:51:00.649 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=entropy 2024-06-18T02:51:00.649 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-06-18T02:51:00.650 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=filefd 2024-06-18T02:51:00.650 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=filesystem 2024-06-18T02:51:00.650 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=hwmon 2024-06-18T02:51:00.650 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=infiniband 2024-06-18T02:51:00.650 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=ipvs 2024-06-18T02:51:00.650 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=loadavg 2024-06-18T02:51:00.650 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=mdadm 2024-06-18T02:51:00.650 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=meminfo 2024-06-18T02:51:00.650 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=netclass 2024-06-18T02:51:00.650 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=netdev 2024-06-18T02:51:00.650 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=netstat 2024-06-18T02:51:00.650 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=nfs 2024-06-18T02:51:00.650 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=nfsd 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=nvme 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=os 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=pressure 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=rapl 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=schedstat 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=sockstat 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=softnet 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=stat 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=tapestats 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=textfile 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=time 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-06-18T02:51:00.651 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=uname 2024-06-18T02:51:00.652 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=vmstat 2024-06-18T02:51:00.652 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=xfs 2024-06-18T02:51:00.652 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.563Z caller=node_exporter.go:115 level=info collector=zfs 2024-06-18T02:51:00.652 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.564Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-06-18T02:51:00.652 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[58786]: ts=2024-06-18T02:51:00.564Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-06-18T02:51:00.652 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 bash[58771]: 8d0aecfcef8a83349c1cdc0291918331c3c0ea8c24f66f604ee3599aa9e25f5f 2024-06-18T02:51:01.042 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 02:51:00 smithi163 systemd[1]: Started Ceph node-exporter.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:51:01.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-mon[29791]: Reconfiguring node-exporter.a (dependencies changed)... 2024-06-18T02:51:01.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-mon[29791]: Reconfiguring daemon node-exporter.a on smithi163 2024-06-18T02:51:01.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-mon[29791]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-18T02:51:01.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:01.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:01.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-mon[25094]: Reconfiguring node-exporter.a (dependencies changed)... 2024-06-18T02:51:01.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-mon[25094]: Reconfiguring daemon node-exporter.a on smithi163 2024-06-18T02:51:01.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-mon[25094]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-18T02:51:01.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:01.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:00 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:01.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:00 smithi186 ceph-mon[25944]: Reconfiguring node-exporter.a (dependencies changed)... 2024-06-18T02:51:01.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:00 smithi186 ceph-mon[25944]: Reconfiguring daemon node-exporter.a on smithi163 2024-06-18T02:51:01.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:00 smithi186 ceph-mon[25944]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-18T02:51:01.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:00 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:01.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:00 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:02.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:01 smithi163 ceph-mon[29791]: Reconfiguring alertmanager.a (dependencies changed)... 2024-06-18T02:51:02.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:01 smithi163 ceph-mon[29791]: Reconfiguring daemon alertmanager.a on smithi163 2024-06-18T02:51:02.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:01 smithi163 ceph-mon[25094]: Reconfiguring alertmanager.a (dependencies changed)... 2024-06-18T02:51:02.180 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:01 smithi163 ceph-mon[25094]: Reconfiguring daemon alertmanager.a on smithi163 2024-06-18T02:51:02.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:01 smithi186 ceph-mon[25944]: Reconfiguring alertmanager.a (dependencies changed)... 2024-06-18T02:51:02.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:01 smithi186 ceph-mon[25944]: Reconfiguring daemon alertmanager.a on smithi163 2024-06-18T02:51:03.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:02 smithi163 ceph-mon[29791]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-18T02:51:03.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:02 smithi163 ceph-mon[25094]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-18T02:51:03.193 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:02 smithi186 ceph-mon[25944]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-18T02:51:03.445 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:51:03] "GET /metrics HTTP/1.1" 200 35103 "" "Prometheus/2.33.4" 2024-06-18T02:51:03.625 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:03 smithi163 systemd[1]: Stopping Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:51:03.625 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:03 smithi163 bash[58959]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager.a" found: no such container 2024-06-18T02:51:03.625 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=error ts=2024-06-18T02:51:03.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:51:03.625 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:51:03.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.163:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.163 because it doesn't contain any IP SANs" 2024-06-18T02:51:03.625 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=warn ts=2024-06-18T02:51:03.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.186:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.186 because it doesn't contain any IP SANs" 2024-06-18T02:51:03.625 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[49767]: level=info ts=2024-06-18T02:51:03.625Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-06-18T02:51:03.957 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:03 smithi163 podman[58969]: 2024-06-18 02:51:03.646247452 +0000 UTC m=+0.120976572 container died 4277853c77458b299a4347b266b69cd74bfe5f59a6365a10c7e12ddee13a832b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:04.251 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:03 smithi163 podman[58969]: 2024-06-18 02:51:03.958661862 +0000 UTC m=+0.433390982 container cleanup 4277853c77458b299a4347b266b69cd74bfe5f59a6365a10c7e12ddee13a832b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:04.251 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:03 smithi163 bash[58969]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a 2024-06-18T02:51:04.251 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:03 smithi163 bash[58997]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager.a" found: no such container 2024-06-18T02:51:04.252 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 podman[58984]: 2024-06-18 02:51:04.193493769 +0000 UTC m=+0.537636640 container remove 4277853c77458b299a4347b266b69cd74bfe5f59a6365a10c7e12ddee13a832b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:04.252 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 podman[58984]: 2024-06-18 02:51:04.252254105 +0000 UTC m=+0.596396932 volume remove 3997a7386d9a69d4098f3364fdec86392c81093900224c1df5c6c0dadd01e97b 2024-06-18T02:51:04.502 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-06-18T02:51:04.502 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 systemd[1]: Stopped Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:51:04.502 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@alertmanager.a.service: Consumed 1.008s CPU time. 2024-06-18T02:51:04.502 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 systemd[1]: Starting Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:51:04.502 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 podman[59059]: 2024-06-18 02:51:04.401476495 +0000 UTC m=+0.017892050 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-06-18T02:51:04.502 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 podman[59059]: 2024-06-18 02:51:04.503119124 +0000 UTC m=+0.119534665 volume create 91054bb2351ff22cb4b6e451b3b4fd0233e60bf1919bdcd624aeda611d619368 2024-06-18T02:51:04.792 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 podman[59059]: 2024-06-18 02:51:04.620358766 +0000 UTC m=+0.236774350 container create 5e109a483ef410ecf5902d8a6eeb474d0b41a09f4181c5528b3983242913747f (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:05.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:04 smithi163 ceph-mon[25094]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T02:51:05.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:04 smithi163 ceph-mon[29791]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T02:51:05.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 podman[59059]: 2024-06-18 02:51:04.889800926 +0000 UTC m=+0.506216482 container init 5e109a483ef410ecf5902d8a6eeb474d0b41a09f4181c5528b3983242913747f (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:05.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 podman[59059]: 2024-06-18 02:51:04.892680641 +0000 UTC m=+0.509096183 container start 5e109a483ef410ecf5902d8a6eeb474d0b41a09f4181c5528b3983242913747f (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:05.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[59075]: level=info ts=2024-06-18T02:51:04.903Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-06-18T02:51:05.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[59075]: level=info ts=2024-06-18T02:51:04.903Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-06-18T02:51:05.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[59075]: level=info ts=2024-06-18T02:51:04.904Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.163 port=9094 2024-06-18T02:51:05.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[59075]: level=info ts=2024-06-18T02:51:04.905Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-06-18T02:51:05.294 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[59075]: level=info ts=2024-06-18T02:51:04.929Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-18T02:51:05.294 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[59075]: level=info ts=2024-06-18T02:51:04.929Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-18T02:51:05.294 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[59075]: level=info ts=2024-06-18T02:51:04.932Z caller=main.go:518 msg=Listening address=:9093 2024-06-18T02:51:05.294 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[59075]: level=info ts=2024-06-18T02:51:04.932Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-06-18T02:51:05.294 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 bash[59059]: 5e109a483ef410ecf5902d8a6eeb474d0b41a09f4181c5528b3983242913747f 2024-06-18T02:51:05.294 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:04 smithi163 systemd[1]: Started Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:51:05.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:04 smithi186 ceph-mon[25944]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T02:51:06.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:06 smithi163 ceph-mon[29791]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:51:06.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:06 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:06.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:06 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:06.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:06 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:51:06.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:06 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:51:06.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:06 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:51:06.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:06 smithi163 ceph-mon[25094]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:51:06.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:06 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:06.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:06 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:06.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:06 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:51:06.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:06 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:51:06.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:06 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:51:06.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:06 smithi186 ceph-mon[25944]: from='client.14817 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:51:06.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:06 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:06.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:06 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:06.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:06 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:51:06.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:06 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:51:06.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:06 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:51:07.099 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:06 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[59075]: level=info ts=2024-06-18T02:51:06.906Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000999388s 2024-06-18T02:51:07.100 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:07 smithi163 ceph-mon[25094]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T02:51:07.100 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:07 smithi163 ceph-mon[25094]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:51:07.100 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:07 smithi163 ceph-mon[25094]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:51:07.100 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:07 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:51:07.100 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:07 smithi163 ceph-mon[29791]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T02:51:07.100 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:07 smithi163 ceph-mon[29791]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:51:07.100 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:07 smithi163 ceph-mon[29791]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:51:07.100 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:07 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:51:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-mon[25944]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T02:51:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-mon[25944]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:51:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-mon[25944]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:51:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:51:08.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:51:07.851+0000 7f957d496640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:51:08.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Creating ceph-iscsi config... 2024-06-18T02:51:08.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:51:08.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:51:08.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:51:08.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:08.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:08.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T02:51:08.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:51:08.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return _run_code(code, main_globals, None, 2024-06-18T02:51:08.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:51:08.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: exec(code, run_globals) 2024-06-18T02:51:08.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:51:08.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:51:08.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:51:08.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:51:08.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:51:08.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:51:08.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:51:08.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:08.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:08.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T02:51:08.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T02:51:08.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.wait_async( 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.event_loop.get_result(coro, timeout) 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return future.result(timeout) 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.__get_result() 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: raise self._exception 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: out, err, code = await self._run_cephadm( 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: raise OrchestratorError( 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:51:08.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Creating ceph-iscsi config... 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return _run_code(code, main_globals, None, 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: exec(code, run_globals) 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:51:08.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:51:08.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:51:08.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:51:08.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:51:08.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:08.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:09.025 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:51:08 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[25322]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:51:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T02:51:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:09 smithi163 ceph-mon[25094]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:51:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:09 smithi163 ceph-mon[25094]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:51:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T02:51:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:51:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:51:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:51:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:09.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:51:09.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:51:09.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T02:51:09.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:51:09.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T02:51:09.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:51:09.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:51:09.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:51:09.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:51:09.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:51:09.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:51:09.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.wait_async( 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: return future.result(timeout) 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.__get_result() 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T02:51:09.295 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise self._exception 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise OrchestratorError( 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:51:09.296 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T02:51:09.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:51:09.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T02:51:09.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:51:09.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:51:09.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:51:09.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:51:09.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:51:09.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:51:09.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:51:09.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:09.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:09.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:09 smithi163 ceph-mon[25094]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T02:51:09.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:09 smithi163 ceph-mon[29791]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:51:09.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:09 smithi163 ceph-mon[29791]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:51:09.298 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T02:51:09.298 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:51:09.298 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:51:09.298 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:51:09.298 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:09.298 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:09.298 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:51:09.298 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:51:09.298 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T02:51:09.298 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:51:09.298 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.wait_async( 2024-06-18T02:51:09.299 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: return future.result(timeout) 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.__get_result() 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise self._exception 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise OrchestratorError( 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:51:09.300 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:51:09.301 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:09.302 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:09.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:09 smithi163 ceph-mon[29791]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T02:51:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:09 smithi186 ceph-mon[25944]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:51:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:09 smithi186 ceph-mon[25944]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:51:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T02:51:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:51:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:51:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:51:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T02:51:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:51:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T02:51:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:51:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T02:51:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:51:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:51:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:51:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:51:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:51:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:51:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:51:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.wait_async( 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: return future.result(timeout) 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.__get_result() 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T02:51:09.320 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise self._exception 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: out, err, code = await self._run_cephadm( 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise OrchestratorError( 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:51:09.321 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T02:51:09.322 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:51:09.322 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T02:51:09.322 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:51:09.322 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:51:09.322 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:51:09.322 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:51:09.322 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:51:09.322 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:51:09.322 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:51:09.322 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:51:09.322 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:51:09.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:09 smithi186 ceph-mon[25944]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T02:51:10.245 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:10 smithi186 ceph-mon[25944]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T02:51:10.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:10 smithi163 ceph-mon[29791]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T02:51:10.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:10 smithi163 ceph-mon[25094]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T02:51:11.066 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:11 smithi186 ceph-mon[25944]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:51:11.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:11 smithi163 ceph-mon[29791]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:51:11.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:11 smithi163 ceph-mon[25094]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:51:11.318 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 systemd[1]: Stopping Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:51:11.318 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 bash[48960]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-18T02:51:11.318 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:51:11.312Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-18T02:51:11.318 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:51:11.312Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-06-18T02:51:11.318 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:51:11.312Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-06-18T02:51:11.318 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:51:11.312Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-06-18T02:51:11.319 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:51:11.312Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-06-18T02:51:11.319 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:51:11.312Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-06-18T02:51:11.319 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:51:11.312Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-18T02:51:11.319 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:51:11.312Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-06-18T02:51:11.319 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:51:11.312Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-06-18T02:51:11.319 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:51:11.312Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-06-18T02:51:11.319 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:51:11.312Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-06-18T02:51:11.319 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[42720]: ts=2024-06-18T02:51:11.313Z caller=main.go:1066 level=info msg="See you next time!" 2024-06-18T02:51:11.595 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 podman[48971]: 2024-06-18 02:51:11.322537359 +0000 UTC m=+0.107155813 container died 782012f610d36a34c191f21a4693fb405c48246d7b04e0cd55724064640f5978 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:11.949 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 podman[48971]: 2024-06-18 02:51:11.686828038 +0000 UTC m=+0.471446489 container cleanup 782012f610d36a34c191f21a4693fb405c48246d7b04e0cd55724064640f5978 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:11.949 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 bash[48971]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a 2024-06-18T02:51:11.949 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 bash[48997]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-18T02:51:12.216 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-mon[25944]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:12.217 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 podman[48986]: 2024-06-18 02:51:11.949302058 +0000 UTC m=+0.628072958 container remove 782012f610d36a34c191f21a4693fb405c48246d7b04e0cd55724064640f5978 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:12.217 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-06-18T02:51:12.217 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 systemd[1]: Stopped Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:51:12.217 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a.service: Consumed 1.908s CPU time. 2024-06-18T02:51:12.217 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:11 smithi186 systemd[1]: Starting Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:51:12.217 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 podman[49062]: 2024-06-18 02:51:12.085369554 +0000 UTC m=+0.018735267 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-06-18T02:51:12.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:12 smithi163 ceph-mon[29791]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:12.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:12 smithi163 ceph-mon[25094]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:12.494 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 podman[49062]: 2024-06-18 02:51:12.217510342 +0000 UTC m=+0.150876091 container create 8512fab6ffb82ede667814a1201132ba8f169fc95482e7a200a35286c445efc4 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:12.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 podman[49062]: 2024-06-18 02:51:12.494644912 +0000 UTC m=+0.428010632 container init 8512fab6ffb82ede667814a1201132ba8f169fc95482e7a200a35286c445efc4 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:12.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 podman[49062]: 2024-06-18 02:51:12.497707959 +0000 UTC m=+0.431073672 container start 8512fab6ffb82ede667814a1201132ba8f169fc95482e7a200a35286c445efc4 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:51:12.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.520Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.520Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.520Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.520Z caller=main.go:518 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi186 (none))" 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.520Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.520Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.522Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.522Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.523Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.525Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.525Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.372µs 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.525Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 bash[49062]: 8512fab6ffb82ede667814a1201132ba8f169fc95482e7a200a35286c445efc4 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 systemd[1]: Started Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:51:12.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.724Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-06-18T02:51:13.289 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:12 smithi186 systemd[1]: Stopping Ceph node-exporter.b for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:51:13.289 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:12 smithi186 bash[49186]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter.b" found: no such container 2024-06-18T02:51:13.289 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:13 smithi186 podman[49196]: 2024-06-18 02:51:13.067174603 +0000 UTC m=+0.132928363 container died 48a19a2881139dae70078a2e6156c9701734fb1c32859d415a7e3699ea00628f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T02:51:13.289 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.960Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-06-18T02:51:13.289 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.961Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-06-18T02:51:13.290 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.961Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=40.487µs wal_replay_duration=436.054953ms total_replay_duration=436.109547ms 2024-06-18T02:51:13.290 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.964Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-06-18T02:51:13.290 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.964Z caller=main.go:947 level=info msg="TSDB started" 2024-06-18T02:51:13.290 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.964Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-18T02:51:13.290 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.985Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.744619ms db_storage=725ns remote_storage=1.29µs web_handler=297ns query_engine=656ns scrape=355.164µs scrape_sd=96.508µs notify=16.155µs notify_sd=15.397µs rules=19.945838ms 2024-06-18T02:51:13.290 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:12.985Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-06-18T02:51:13.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:13 smithi186 podman[49196]: 2024-06-18 02:51:13.481923961 +0000 UTC m=+0.547677720 container cleanup 48a19a2881139dae70078a2e6156c9701734fb1c32859d415a7e3699ea00628f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T02:51:13.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:13 smithi186 bash[49196]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b 2024-06-18T02:51:13.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:13 smithi186 bash[49222]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter.b" found: no such container 2024-06-18T02:51:13.923 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:13 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:13.924 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:13 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:13.924 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:13 smithi186 ceph-mon[25944]: Reconfiguring node-exporter.b (dependencies changed)... 2024-06-18T02:51:13.924 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:13 smithi186 ceph-mon[25944]: Reconfiguring daemon node-exporter.b on smithi186 2024-06-18T02:51:13.924 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:13 smithi186 podman[49208]: 2024-06-18 02:51:13.699988948 +0000 UTC m=+0.633458496 container remove 48a19a2881139dae70078a2e6156c9701734fb1c32859d415a7e3699ea00628f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T02:51:13.924 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:13 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-06-18T02:51:13.924 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:13 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-06-18T02:51:13.924 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:13 smithi186 systemd[1]: Stopped Ceph node-exporter.b for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:51:13.924 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:13 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.b.service: Consumed 3.736s CPU time. 2024-06-18T02:51:13.925 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:13 smithi186 systemd[1]: Starting Ceph node-exporter.b for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:51:14.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:13 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:13 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:13 smithi163 ceph-mon[29791]: Reconfiguring node-exporter.b (dependencies changed)... 2024-06-18T02:51:14.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:13 smithi163 ceph-mon[29791]: Reconfiguring daemon node-exporter.b on smithi186 2024-06-18T02:51:14.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:13 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:13 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:13 smithi163 ceph-mon[25094]: Reconfiguring node-exporter.b (dependencies changed)... 2024-06-18T02:51:14.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:13 smithi163 ceph-mon[25094]: Reconfiguring daemon node-exporter.b on smithi186 2024-06-18T02:51:14.238 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:13 smithi186 podman[49283]: 2024-06-18 02:51:13.826245703 +0000 UTC m=+0.016915236 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-06-18T02:51:14.238 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:13 smithi186 podman[49283]: 2024-06-18 02:51:13.9484467 +0000 UTC m=+0.139116219 container create 0592fede9474f20dd00e22f4bfe87cb492f4879ef1567fa785bdcf9b4249e59c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T02:51:14.239 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 podman[49283]: 2024-06-18 02:51:14.138374397 +0000 UTC m=+0.329043936 container init 0592fede9474f20dd00e22f4bfe87cb492f4879ef1567fa785bdcf9b4249e59c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T02:51:14.239 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 podman[49283]: 2024-06-18 02:51:14.141180714 +0000 UTC m=+0.331850245 container start 0592fede9474f20dd00e22f4bfe87cb492f4879ef1567fa785bdcf9b4249e59c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T02:51:14.239 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.144Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-06-18T02:51:14.239 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.144Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-06-18T02:51:14.239 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.144Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-06-18T02:51:14.239 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-06-18T02:51:14.239 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-06-18T02:51:14.239 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=arp 2024-06-18T02:51:14.239 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=bcache 2024-06-18T02:51:14.239 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=bonding 2024-06-18T02:51:14.239 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=btrfs 2024-06-18T02:51:14.239 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=conntrack 2024-06-18T02:51:14.239 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=cpu 2024-06-18T02:51:14.240 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-06-18T02:51:14.240 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=diskstats 2024-06-18T02:51:14.240 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=dmi 2024-06-18T02:51:14.240 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=edac 2024-06-18T02:51:14.240 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=entropy 2024-06-18T02:51:14.240 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-06-18T02:51:14.240 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=filefd 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=filesystem 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=hwmon 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=infiniband 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=ipvs 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=loadavg 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=mdadm 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=meminfo 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=netclass 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=netdev 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=netstat 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=nfs 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=nfsd 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=nvme 2024-06-18T02:51:14.241 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=os 2024-06-18T02:51:14.242 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-06-18T02:51:14.242 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=pressure 2024-06-18T02:51:14.242 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=rapl 2024-06-18T02:51:14.242 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=schedstat 2024-06-18T02:51:14.242 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=sockstat 2024-06-18T02:51:14.242 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=softnet 2024-06-18T02:51:14.242 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=stat 2024-06-18T02:51:14.242 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=tapestats 2024-06-18T02:51:14.242 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=textfile 2024-06-18T02:51:14.242 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-06-18T02:51:14.242 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=time 2024-06-18T02:51:14.242 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-06-18T02:51:14.242 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=uname 2024-06-18T02:51:14.243 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=vmstat 2024-06-18T02:51:14.243 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=xfs 2024-06-18T02:51:14.243 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:115 level=info collector=zfs 2024-06-18T02:51:14.243 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-06-18T02:51:14.243 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[49298]: ts=2024-06-18T02:51:14.145Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-06-18T02:51:14.243 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 bash[49283]: 0592fede9474f20dd00e22f4bfe87cb492f4879ef1567fa785bdcf9b4249e59c 2024-06-18T02:51:14.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 02:51:14 smithi186 systemd[1]: Started Ceph node-exporter.b for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:51:14.893 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-mon[25944]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:14.893 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.894 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.894 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.894 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.894 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:14 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-18T02:51:14.907 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:14 smithi163 ceph-mon[29791]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:14.908 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:14 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.908 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:14 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.908 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:14 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.908 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:14 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.908 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:14 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-18T02:51:14.908 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:14 smithi163 ceph-mon[25094]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:14.908 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:14 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.908 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:14 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.908 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:14 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.908 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:14 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:14.909 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:14 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-18T02:51:15.292 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 02:51:14 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[59075]: level=info ts=2024-06-18T02:51:14.908Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002856857s 2024-06-18T02:51:15.871 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:15 smithi186 ceph-mon[25944]: Reconfiguring grafana.a (dependencies changed)... 2024-06-18T02:51:15.871 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:15 smithi186 ceph-mon[25944]: Regenerating cephadm self-signed grafana TLS certificates 2024-06-18T02:51:15.871 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:15 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-18T02:51:15.871 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:15 smithi186 ceph-mon[25944]: Reconfiguring daemon grafana.a on smithi186 2024-06-18T02:51:16.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:15 smithi163 ceph-mon[29791]: Reconfiguring grafana.a (dependencies changed)... 2024-06-18T02:51:16.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:15 smithi163 ceph-mon[29791]: Regenerating cephadm self-signed grafana TLS certificates 2024-06-18T02:51:16.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:15 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-18T02:51:16.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:15 smithi163 ceph-mon[29791]: Reconfiguring daemon grafana.a on smithi186 2024-06-18T02:51:16.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:15 smithi163 ceph-mon[25094]: Reconfiguring grafana.a (dependencies changed)... 2024-06-18T02:51:16.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:15 smithi163 ceph-mon[25094]: Regenerating cephadm self-signed grafana TLS certificates 2024-06-18T02:51:16.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:15 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-18T02:51:16.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:15 smithi163 ceph-mon[25094]: Reconfiguring daemon grafana.a on smithi186 2024-06-18T02:51:16.961 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:16 smithi186 ceph-mon[25944]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-18T02:51:17.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:16 smithi163 ceph-mon[29791]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-18T02:51:17.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:16 smithi163 ceph-mon[25094]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-18T02:51:17.318 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:17 smithi186 systemd[1]: Stopping Ceph grafana.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:51:17.318 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:17 smithi186 bash[49468]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana.a" found: no such container 2024-06-18T02:51:17.318 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[41942]: t=2024-06-18T02:51:17+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-06-18T02:51:17.318 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:17 smithi186 podman[49478]: 2024-06-18 02:51:17.183105621 +0000 UTC m=+0.136330593 container died f134c5a8fb6d01a86deb8a35b23d279f2a7c5a8d79a1589fcee979c20615ef51 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., distribution-scope=public, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, architecture=x86_64, io.buildah.version=1.24.2, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, build-date=2022-03-28T10:36:18.413762) 2024-06-18T02:51:17.818 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:17 smithi186 podman[49478]: 2024-06-18 02:51:17.539481182 +0000 UTC m=+0.492706152 container cleanup f134c5a8fb6d01a86deb8a35b23d279f2a7c5a8d79a1589fcee979c20615ef51 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2, build-date=2022-03-28T10:36:18.413762, name=ubi8, release=236.1648460182, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, architecture=x86_64, description=Ceph Grafana Container, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-type=git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, vendor=Red Hat, Inc.) 2024-06-18T02:51:17.818 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:17 smithi186 bash[49478]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a 2024-06-18T02:51:17.818 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:17 smithi186 bash[49504]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana.a" found: no such container 2024-06-18T02:51:18.227 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:17 smithi186 podman[49492]: 2024-06-18 02:51:17.832868219 +0000 UTC m=+0.649564728 container remove f134c5a8fb6d01a86deb8a35b23d279f2a7c5a8d79a1589fcee979c20615ef51 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, name=ubi8, distribution-scope=public, architecture=x86_64, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, release=236.1648460182, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.openshift.expose-services=, com.redhat.component=ubi8-container, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-06-18T02:51:18.227 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:17 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@grafana.a.service: Deactivated successfully. 2024-06-18T02:51:18.227 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:17 smithi186 systemd[1]: Stopped Ceph grafana.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:51:18.227 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:17 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@grafana.a.service: Consumed 3.261s CPU time. 2024-06-18T02:51:18.227 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:17 smithi186 systemd[1]: Starting Ceph grafana.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:51:18.227 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 podman[49566]: 2024-06-18 02:51:17.959167689 +0000 UTC m=+0.016741162 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-06-18T02:51:18.228 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 podman[49566]: 2024-06-18 02:51:18.058738857 +0000 UTC m=+0.116312332 container create a3fd9ade837492165c6d08920beb7df7fc34e05e2fd9a58e6194129517d04df1 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., release=236.1648460182, io.openshift.tags=base rhel8, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, distribution-scope=public, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2, maintainer=Paul Cuzner ) 2024-06-18T02:51:18.492 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 podman[49566]: 2024-06-18 02:51:18.428060994 +0000 UTC m=+0.485634471 container init a3fd9ade837492165c6d08920beb7df7fc34e05e2fd9a58e6194129517d04df1 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, build-date=2022-03-28T10:36:18.413762, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, com.redhat.component=ubi8-container, vcs-type=git, architecture=x86_64, io.buildah.version=1.24.2, release=236.1648460182, vendor=Red Hat, Inc., distribution-scope=public, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8) 2024-06-18T02:51:18.492 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 podman[49566]: 2024-06-18 02:51:18.431081804 +0000 UTC m=+0.488655276 container start a3fd9ade837492165c6d08920beb7df7fc34e05e2fd9a58e6194129517d04df1 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, description=Ceph Grafana Container, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2, name=ubi8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-type=git, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, version=8.5, io.openshift.expose-services=, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., release=236.1648460182, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-06-18T02:51:18.492 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-06-18T02:51:18.492 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-06-18T02:51:18.492 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-06-18T02:51:18.492 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-06-18T02:51:18.492 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-06-18T02:51:18.492 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-06-18T02:51:18.492 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="App mode production" logger=settings 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=530.515µs 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-06-18T02:51:18.493 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-06-18T02:51:18.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:18] ENGINE Bus STOPPING 2024-06-18T02:51:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-mon[25944]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-18T02:51:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T02:51:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2024-06-18T02:51:18.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-06-18T02:51:18.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-06-18T02:51:18.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-06-18T02:51:18.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 bash[49566]: a3fd9ade837492165c6d08920beb7df7fc34e05e2fd9a58e6194129517d04df1 2024-06-18T02:51:18.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 systemd[1]: Started Ceph grafana.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:51:18.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-06-18T02:51:18.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-06-18T02:51:18.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-06-18T02:51:18.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-06-18T02:51:18.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-06-18T02:51:18.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T02:51:18+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-06-18T02:51:18.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:18 smithi163 ceph-mon[29791]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-18T02:51:18.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:18 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:18.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:18 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:18.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:18 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T02:51:18.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:18 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2024-06-18T02:51:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:18 smithi163 ceph-mon[25094]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-18T02:51:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:18 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:18.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:18 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:18.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:18 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T02:51:18.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:18 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2024-06-18T02:51:19.316 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-18T02:51:19.317 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:18] ENGINE Bus STOPPED 2024-06-18T02:51:19.317 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:18] ENGINE Bus STARTING 2024-06-18T02:51:19.317 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:18] ENGINE Serving on http://:::9283 2024-06-18T02:51:19.317 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:18] ENGINE Bus STARTED 2024-06-18T02:51:19.317 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:18] ENGINE Bus STOPPING 2024-06-18T02:51:19.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-18T02:51:19.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:19] ENGINE Bus STOPPED 2024-06-18T02:51:19.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:19] ENGINE Bus STARTING 2024-06-18T02:51:19.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:19] ENGINE Serving on http://:::9283 2024-06-18T02:51:19.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:19] ENGINE Bus STARTED 2024-06-18T02:51:19.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:19] ENGINE Bus STOPPING 2024-06-18T02:51:19.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T02:51:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2024-06-18T02:51:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:51:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:51:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi186.front.sepia.ceph.com:9095"}]: dispatch 2024-06-18T02:51:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi186.front.sepia.ceph.com:9095"}]: dispatch 2024-06-18T02:51:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T02:51:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T02:51:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi186.front.sepia.ceph.com:3000"}]: dispatch 2024-06-18T02:51:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi186.front.sepia.ceph.com:3000"}]: dispatch 2024-06-18T02:51:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1009084079' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T02:51:19.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T02:51:19.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2024-06-18T02:51:19.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:19.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:51:19.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:51:19.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi186.front.sepia.ceph.com:9095"}]: dispatch 2024-06-18T02:51:19.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi186.front.sepia.ceph.com:9095"}]: dispatch 2024-06-18T02:51:19.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:19.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T02:51:19.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T02:51:19.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi186.front.sepia.ceph.com:3000"}]: dispatch 2024-06-18T02:51:19.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi186.front.sepia.ceph.com:3000"}]: dispatch 2024-06-18T02:51:19.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:19.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:19 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1009084079' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T02:51:20.036 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T02:51:20.036 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2024-06-18T02:51:20.036 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:20.036 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:51:20.036 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:51:20.036 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi186.front.sepia.ceph.com:9095"}]: dispatch 2024-06-18T02:51:20.036 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi186.front.sepia.ceph.com:9095"}]: dispatch 2024-06-18T02:51:20.036 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:20.037 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T02:51:20.037 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T02:51:20.037 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi186.front.sepia.ceph.com:3000"}]: dispatch 2024-06-18T02:51:20.037 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi186.front.sepia.ceph.com:3000"}]: dispatch 2024-06-18T02:51:20.037 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:20.037 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:19 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1009084079' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T02:51:20.287 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:20 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-18T02:51:20.287 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:20 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:20] ENGINE Bus STOPPED 2024-06-18T02:51:20.287 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:20 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:20] ENGINE Bus STARTING 2024-06-18T02:51:20.287 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:20 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:20] ENGINE Serving on http://:::9283 2024-06-18T02:51:20.287 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:20 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:51:20] ENGINE Bus STARTED 2024-06-18T02:51:20.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:20 smithi163 ceph-mon[25094]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-18T02:51:20.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:20 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/4247559095' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3786129084"}]: dispatch 2024-06-18T02:51:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:20 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3786129084"}]: dispatch 2024-06-18T02:51:20.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:20 smithi163 ceph-mon[29791]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-18T02:51:20.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:20 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/4247559095' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3786129084"}]: dispatch 2024-06-18T02:51:20.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:20 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3786129084"}]: dispatch 2024-06-18T02:51:20.938 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:20 smithi186 ceph-mon[25944]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-18T02:51:20.939 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:20 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/4247559095' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3786129084"}]: dispatch 2024-06-18T02:51:20.939 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:20 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3786129084"}]: dispatch 2024-06-18T02:51:21.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:21 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3786129084"}]': finished 2024-06-18T02:51:21.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:21 smithi163 ceph-mon[29791]: osdmap e96: 8 total, 8 up, 8 in 2024-06-18T02:51:21.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:21 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2186986534' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3854405794"}]: dispatch 2024-06-18T02:51:21.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:21 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3786129084"}]': finished 2024-06-18T02:51:21.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:21 smithi163 ceph-mon[25094]: osdmap e96: 8 total, 8 up, 8 in 2024-06-18T02:51:21.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:21 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2186986534' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3854405794"}]: dispatch 2024-06-18T02:51:22.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:21 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3786129084"}]': finished 2024-06-18T02:51:22.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:21 smithi186 ceph-mon[25944]: osdmap e96: 8 total, 8 up, 8 in 2024-06-18T02:51:22.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:21 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2186986534' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3854405794"}]: dispatch 2024-06-18T02:51:22.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:51:22] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-06-18T02:51:23.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[25094]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:51:23.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2186986534' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3854405794"}]': finished 2024-06-18T02:51:23.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[25094]: osdmap e97: 8 total, 8 up, 8 in 2024-06-18T02:51:23.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:51:23.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1968760519' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/3786129084"}]: dispatch 2024-06-18T02:51:23.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:23.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[29791]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:51:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2186986534' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3854405794"}]': finished 2024-06-18T02:51:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[29791]: osdmap e97: 8 total, 8 up, 8 in 2024-06-18T02:51:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:51:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1968760519' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/3786129084"}]: dispatch 2024-06-18T02:51:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:22 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:23.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:22 smithi186 ceph-mon[25944]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:51:23.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:22 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2186986534' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3854405794"}]': finished 2024-06-18T02:51:23.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:22 smithi186 ceph-mon[25944]: osdmap e97: 8 total, 8 up, 8 in 2024-06-18T02:51:23.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:22 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:51:23.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:22 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1968760519' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/3786129084"}]: dispatch 2024-06-18T02:51:23.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:22 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:23.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:22 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:23.682 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:51:24.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1968760519' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/3786129084"}]': finished 2024-06-18T02:51:24.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: osdmap e98: 8 total, 8 up, 8 in 2024-06-18T02:51:24.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:24.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:24.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:51:24.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:51:24.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:24.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T02:51:24.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:51:24.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:51:24.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:51:24.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:51:24.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3014581256' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3313286018"}]: dispatch 2024-06-18T02:51:24.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3313286018"}]: dispatch 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1968760519' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/3786129084"}]': finished 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: osdmap e98: 8 total, 8 up, 8 in 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3014581256' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3313286018"}]: dispatch 2024-06-18T02:51:24.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:23 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3313286018"}]: dispatch 2024-06-18T02:51:24.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1968760519' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/3786129084"}]': finished 2024-06-18T02:51:24.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: osdmap e98: 8 total, 8 up, 8 in 2024-06-18T02:51:24.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:24.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:24.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:51:24.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:51:24.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:24.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T02:51:24.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:51:24.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:51:24.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:51:24.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:51:24.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3014581256' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3313286018"}]: dispatch 2024-06-18T02:51:24.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:23 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3313286018"}]: dispatch 2024-06-18T02:51:25.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:24 smithi163 ceph-mon[25094]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-06-18T02:51:25.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:24 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3313286018"}]': finished 2024-06-18T02:51:25.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:24 smithi163 ceph-mon[25094]: osdmap e99: 8 total, 8 up, 8 in 2024-06-18T02:51:25.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:24 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2646531562' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/4111534652"}]: dispatch 2024-06-18T02:51:25.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:24 smithi163 ceph-mon[29791]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-06-18T02:51:25.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:24 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3313286018"}]': finished 2024-06-18T02:51:25.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:24 smithi163 ceph-mon[29791]: osdmap e99: 8 total, 8 up, 8 in 2024-06-18T02:51:25.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:24 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2646531562' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/4111534652"}]: dispatch 2024-06-18T02:51:25.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:24 smithi186 ceph-mon[25944]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-06-18T02:51:25.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:24 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3313286018"}]': finished 2024-06-18T02:51:25.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:24 smithi186 ceph-mon[25944]: osdmap e99: 8 total, 8 up, 8 in 2024-06-18T02:51:25.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:24 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2646531562' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/4111534652"}]: dispatch 2024-06-18T02:51:26.022 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:25 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2646531562' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/4111534652"}]': finished 2024-06-18T02:51:26.023 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:25 smithi163 ceph-mon[29791]: osdmap e100: 8 total, 8 up, 8 in 2024-06-18T02:51:26.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:25 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2646531562' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/4111534652"}]': finished 2024-06-18T02:51:26.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:25 smithi163 ceph-mon[25094]: osdmap e100: 8 total, 8 up, 8 in 2024-06-18T02:51:26.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:25 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2646531562' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/4111534652"}]': finished 2024-06-18T02:51:26.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:25 smithi186 ceph-mon[25944]: osdmap e100: 8 total, 8 up, 8 in 2024-06-18T02:51:26.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:26 smithi163 ceph-mon[29791]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-18T02:51:26.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:26 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:26.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:26 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:26.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:26 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:51:26.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:26 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:51:26.884 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:26 smithi163 ceph-mon[25094]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-18T02:51:26.884 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:26 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:26.884 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:26 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:26.884 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:26 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:51:26.884 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:26 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:51:27.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:26 smithi186 ceph-mon[25944]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-18T02:51:27.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:26 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:27.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:26 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:27.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:26 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:51:27.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:26 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:51:28.214 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:27 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:51:28.215 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:27 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:51:28.215 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:27 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/10075495' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T02:51:28.215 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:27 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1246463776' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1481594701"}]: dispatch 2024-06-18T02:51:28.215 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:27 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1481594701"}]: dispatch 2024-06-18T02:51:28.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:27 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:51:28.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:27 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:51:28.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:27 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/10075495' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T02:51:28.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:27 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1246463776' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1481594701"}]: dispatch 2024-06-18T02:51:28.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:27 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1481594701"}]: dispatch 2024-06-18T02:51:28.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:27 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:51:28.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:27 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:51:28.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:27 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/10075495' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T02:51:28.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:27 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1246463776' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1481594701"}]: dispatch 2024-06-18T02:51:28.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:27 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1481594701"}]: dispatch 2024-06-18T02:51:29.000 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:28 smithi186 ceph-mon[25944]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 713 B/s rd, 0 op/s 2024-06-18T02:51:29.000 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:28 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1481594701"}]': finished 2024-06-18T02:51:29.000 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:28 smithi186 ceph-mon[25944]: osdmap e101: 8 total, 8 up, 8 in 2024-06-18T02:51:29.000 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:28 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3459328987' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3601932060"}]: dispatch 2024-06-18T02:51:29.000 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:51:29.186 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:28 smithi163 ceph-mon[25094]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 713 B/s rd, 0 op/s 2024-06-18T02:51:29.186 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:28 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1481594701"}]': finished 2024-06-18T02:51:29.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:28 smithi163 ceph-mon[25094]: osdmap e101: 8 total, 8 up, 8 in 2024-06-18T02:51:29.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:28 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3459328987' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3601932060"}]: dispatch 2024-06-18T02:51:29.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:28 smithi163 ceph-mon[29791]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 713 B/s rd, 0 op/s 2024-06-18T02:51:29.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:28 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/1481594701"}]': finished 2024-06-18T02:51:29.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:28 smithi163 ceph-mon[29791]: osdmap e101: 8 total, 8 up, 8 in 2024-06-18T02:51:29.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:28 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3459328987' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3601932060"}]: dispatch 2024-06-18T02:51:30.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:29 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3459328987' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3601932060"}]': finished 2024-06-18T02:51:30.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:29 smithi163 ceph-mon[25094]: osdmap e102: 8 total, 8 up, 8 in 2024-06-18T02:51:30.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:29 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3459328987' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3601932060"}]': finished 2024-06-18T02:51:30.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:29 smithi163 ceph-mon[29791]: osdmap e102: 8 total, 8 up, 8 in 2024-06-18T02:51:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:29 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3459328987' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3601932060"}]': finished 2024-06-18T02:51:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:29 smithi186 ceph-mon[25944]: osdmap e102: 8 total, 8 up, 8 in 2024-06-18T02:51:31.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[29791]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 738 B/s rd, 1 op/s 2024-06-18T02:51:31.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:51:31.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:51:31.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[25094]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 738 B/s rd, 1 op/s 2024-06-18T02:51:31.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:51:31.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:51:31.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:30 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:30 smithi186 ceph-mon[25944]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 738 B/s rd, 1 op/s 2024-06-18T02:51:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:30 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:30 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:30 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:30 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:30 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:51:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:30 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:51:31.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:30 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:51:32.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:51:32] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-06-18T02:51:33.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:32 smithi163 ceph-mon[29791]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 626 B/s rd, 0 op/s 2024-06-18T02:51:33.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:32 smithi163 ceph-mon[25094]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 626 B/s rd, 0 op/s 2024-06-18T02:51:33.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:32 smithi186 ceph-mon[25944]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 626 B/s rd, 0 op/s 2024-06-18T02:51:33.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:33.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:51:35.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:34 smithi163 ceph-mon[29791]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T02:51:35.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:34 smithi163 ceph-mon[25094]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T02:51:35.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:34 smithi186 ceph-mon[25944]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T02:51:36.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:36 smithi163 ceph-mon[29791]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T02:51:36.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:36 smithi163 ceph-mon[25094]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T02:51:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:36 smithi186 ceph-mon[25944]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T02:51:37.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:37 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:51:37.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:37 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:51:37.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:37 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:51:37.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:37 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:51:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:37 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:51:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:37 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:51:38.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:38 smithi163 ceph-mon[29791]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 537 B/s rd, 0 op/s 2024-06-18T02:51:38.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:38 smithi163 ceph-mon[25094]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 537 B/s rd, 0 op/s 2024-06-18T02:51:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:38 smithi186 ceph-mon[25944]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 537 B/s rd, 0 op/s 2024-06-18T02:51:38.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:51:40.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:40 smithi163 ceph-mon[29791]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 0 op/s 2024-06-18T02:51:40.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:40 smithi163 ceph-mon[25094]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 0 op/s 2024-06-18T02:51:40.817 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:40 smithi186 ceph-mon[25944]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 0 op/s 2024-06-18T02:51:42.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:42 smithi186 ceph-mon[25944]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:42.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:51:42] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-06-18T02:51:42.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:42 smithi163 ceph-mon[29791]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:42.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:42 smithi163 ceph-mon[25094]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:43.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:51:44.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:44 smithi163 ceph-mon[25094]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:51:44.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:44 smithi163 ceph-mon[29791]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:51:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:44 smithi186 ceph-mon[25944]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:51:46.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:46 smithi163 ceph-mon[29791]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:46.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:46 smithi163 ceph-mon[25094]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:46 smithi186 ceph-mon[25944]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:47.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:47 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:51:47.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:47 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:51:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:47 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:51:48.659 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:48 smithi186 ceph-mon[25944]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:48.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:48 smithi163 ceph-mon[29791]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:48.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:48 smithi163 ceph-mon[25094]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:49.068 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:51:50.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:50 smithi163 ceph-mon[29791]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:51:50.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:50 smithi163 ceph-mon[25094]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:51:50.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:50 smithi186 ceph-mon[25944]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:51:52.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:52 smithi186 ceph-mon[25944]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:52.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:52 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:51:52.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:51:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:51:52] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-06-18T02:51:52.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:52 smithi163 ceph-mon[29791]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:52.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:52 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:51:52.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:52 smithi163 ceph-mon[25094]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:52.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:52 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:51:53.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:51:54.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:54 smithi163 ceph-mon[25094]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:51:54.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:54 smithi163 ceph-mon[29791]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:51:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:54 smithi186 ceph-mon[25944]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:51:56.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:56 smithi163 ceph-mon[25094]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:56.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:56 smithi163 ceph-mon[29791]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:56 smithi186 ceph-mon[25944]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:57.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:57 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:51:57.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:57 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:51:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:57 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:51:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:51:58 smithi163 ceph-mon[29791]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:58.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:51:58 smithi163 ceph-mon[25094]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:51:58 smithi186 ceph-mon[25944]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:51:58.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:51:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:51:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:52:00.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:00 smithi163 ceph-mon[29791]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:00.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:00 smithi163 ceph-mon[25094]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:00 smithi186 ceph-mon[25944]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:02 smithi186 ceph-mon[25944]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:02.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:52:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:52:02] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-06-18T02:52:02.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:02 smithi163 ceph-mon[29791]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:02.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:02 smithi163 ceph-mon[25094]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:03.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:52:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:52:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:52:04.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:04 smithi163 ceph-mon[29791]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:04.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:04 smithi163 ceph-mon[25094]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:04.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:04 smithi186 ceph-mon[25944]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:06.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:06 smithi163 ceph-mon[25094]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:06.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:06 smithi163 ceph-mon[29791]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:06 smithi186 ceph-mon[25944]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:07.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:07 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:52:07.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:07 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:07.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:07 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:52:07.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:07 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:07 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:52:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:07 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:08.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:08 smithi163 ceph-mon[25094]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:08.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:08 smithi163 ceph-mon[29791]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:08 smithi186 ceph-mon[25944]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:08.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:52:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:52:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:52:10.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:10 smithi163 ceph-mon[25094]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:10.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:10 smithi163 ceph-mon[29791]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:10.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:10 smithi186 ceph-mon[25944]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:12.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:12 smithi186 ceph-mon[25944]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:12.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:52:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:52:12] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-06-18T02:52:12.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:12 smithi163 ceph-mon[29791]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:12.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:12 smithi163 ceph-mon[25094]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:13.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:52:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:52:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:52:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:14 smithi163 ceph-mon[29791]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:14.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:14 smithi163 ceph-mon[25094]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:14 smithi186 ceph-mon[25944]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:16.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:16 smithi163 ceph-mon[29791]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:16 smithi163 ceph-mon[25094]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:16 smithi186 ceph-mon[25944]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:17.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:17 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:17.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:17 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:17 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:18.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:18 smithi163 ceph-mon[25094]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:18.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:18 smithi163 ceph-mon[29791]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:18 smithi186 ceph-mon[25944]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:18.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:52:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:52:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:52:20.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:20 smithi163 ceph-mon[25094]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:20.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:20 smithi163 ceph-mon[29791]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:20.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:20 smithi186 ceph-mon[25944]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:22.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:22 smithi186 ceph-mon[25944]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:22.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:22 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:52:22.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:52:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:52:22] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-06-18T02:52:22.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:22 smithi163 ceph-mon[25094]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:22.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:22 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:52:22.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:22 smithi163 ceph-mon[29791]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:22.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:22 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:52:23.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:52:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:52:23.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:52:24.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:24 smithi163 ceph-mon[29791]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:24 smithi163 ceph-mon[25094]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:24 smithi186 ceph-mon[25944]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:26.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:26 smithi163 ceph-mon[29791]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:26 smithi163 ceph-mon[25094]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:26 smithi186 ceph-mon[25944]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:27.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:27 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:27.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:27 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:27 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:28.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:28 smithi163 ceph-mon[29791]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:28 smithi163 ceph-mon[25094]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:28 smithi186 ceph-mon[25944]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:28.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:52:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:52:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:52:30.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:30 smithi163 ceph-mon[29791]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:30 smithi163 ceph-mon[25094]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:30 smithi186 ceph-mon[25944]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:32.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:32 smithi186 ceph-mon[25944]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:32.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:52:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:52:32] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-06-18T02:52:32.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:32 smithi163 ceph-mon[29791]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:32.772 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:32 smithi163 ceph-mon[25094]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:33.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:52:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:52:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:52:34.760 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:34 smithi186 ceph-mon[25944]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:34.760 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:34 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:34.760 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:34 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:34.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:34 smithi163 ceph-mon[25094]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:34.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:34 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:34.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:34 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:34.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:34 smithi163 ceph-mon[29791]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:34.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:34 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:34.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:34 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:36.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:36 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:36.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:36 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:36.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:36 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:52:36.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:36 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:52:36.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:36 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:36.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:36 smithi186 ceph-mon[25944]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:36.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:36 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:36.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:36 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:36.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:36 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:52:36.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:36 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:52:36.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:36 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:36.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:36 smithi163 ceph-mon[25094]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:36.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:36 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:36.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:36 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:36.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:36 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:52:36.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:36 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:52:36.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:36 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:52:36.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:36 smithi163 ceph-mon[29791]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:37.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:37 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:52:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:37 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:37.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:37 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:52:37.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:37 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:37 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:52:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:37 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:38.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:38 smithi163 ceph-mon[29791]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:38 smithi163 ceph-mon[25094]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:38 smithi186 ceph-mon[25944]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:38.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:52:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:52:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:52:40.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:40 smithi163 ceph-mon[29791]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:40.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:40 smithi163 ceph-mon[25094]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:40 smithi186 ceph-mon[25944]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:42.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:42 smithi186 ceph-mon[25944]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:42.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:52:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:52:42] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-06-18T02:52:42.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:42 smithi163 ceph-mon[29791]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:42.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:42 smithi163 ceph-mon[25094]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:43.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:52:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:52:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:52:44.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:44 smithi163 ceph-mon[29791]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:44 smithi163 ceph-mon[25094]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:44 smithi186 ceph-mon[25944]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:46.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:46 smithi163 ceph-mon[29791]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:46 smithi163 ceph-mon[25094]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:46 smithi186 ceph-mon[25944]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:47.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:47 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:47.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:47 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:47 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:48.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:48 smithi163 ceph-mon[29791]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:48.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:48 smithi163 ceph-mon[25094]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:48 smithi186 ceph-mon[25944]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:48.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:52:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:52:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:52:50.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:50 smithi163 ceph-mon[29791]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:50.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:50 smithi163 ceph-mon[25094]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:50.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:50 smithi186 ceph-mon[25944]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:52.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:52 smithi186 ceph-mon[25944]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:52.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:52 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:52:52.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:52:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:52:52] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-06-18T02:52:52.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:52 smithi163 ceph-mon[29791]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:52.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:52 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:52:52.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:52 smithi163 ceph-mon[25094]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:52.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:52 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:52:53.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:52:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:52:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:52:54.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:54 smithi163 ceph-mon[29791]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:54.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:54 smithi163 ceph-mon[25094]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:54 smithi186 ceph-mon[25944]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:52:56.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:56 smithi163 ceph-mon[25094]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:56.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:56 smithi163 ceph-mon[29791]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:56 smithi186 ceph-mon[25944]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:57.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:57 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:57.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:57 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:57 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:52:58.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:52:58 smithi163 ceph-mon[29791]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:58.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:52:58 smithi163 ceph-mon[25094]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:52:58 smithi186 ceph-mon[25944]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:52:58.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:52:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:52:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:53:00.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:00 smithi163 ceph-mon[29791]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:00.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:00 smithi163 ceph-mon[25094]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:00 smithi186 ceph-mon[25944]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:02 smithi186 ceph-mon[25944]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:02.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:53:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:53:02] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-06-18T02:53:02.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:02 smithi163 ceph-mon[29791]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:02 smithi163 ceph-mon[25094]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:03.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:53:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:53:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:53:04.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:04 smithi163 ceph-mon[29791]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:04.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:04 smithi163 ceph-mon[25094]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:04.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:04 smithi186 ceph-mon[25944]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:06.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:06 smithi163 ceph-mon[29791]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:06.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:06 smithi163 ceph-mon[25094]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:06 smithi186 ceph-mon[25944]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:07.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:07 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:53:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:07 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:07.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:07 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:53:07.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:07 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:07 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:53:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:07 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:08 smithi163 ceph-mon[29791]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:08.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:08 smithi163 ceph-mon[25094]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:08 smithi186 ceph-mon[25944]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:08.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:53:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:53:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:53:10.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:10 smithi163 ceph-mon[25094]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:10.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:10 smithi163 ceph-mon[29791]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:10.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:10 smithi186 ceph-mon[25944]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:12.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:12 smithi186 ceph-mon[25944]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:12.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:53:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:53:12] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-18T02:53:12.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:12 smithi163 ceph-mon[29791]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:12.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:12 smithi163 ceph-mon[25094]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:13.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:53:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:53:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:53:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:14 smithi163 ceph-mon[29791]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:14 smithi163 ceph-mon[25094]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:14 smithi186 ceph-mon[25944]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:16.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:16 smithi163 ceph-mon[29791]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:16 smithi163 ceph-mon[25094]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:16 smithi186 ceph-mon[25944]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:17.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:17 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:17.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:17 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:17 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:18.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:18 smithi163 ceph-mon[29791]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:18.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:18 smithi163 ceph-mon[25094]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:18 smithi186 ceph-mon[25944]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:18.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:53:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:53:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:53:20.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:20 smithi163 ceph-mon[29791]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:20 smithi163 ceph-mon[25094]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:20.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:20 smithi186 ceph-mon[25944]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:22.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:22 smithi186 ceph-mon[25944]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:22.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:22 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:53:22.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:53:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:53:22] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.33.4" 2024-06-18T02:53:22.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:22 smithi163 ceph-mon[29791]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:22.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:22 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:53:22.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:22 smithi163 ceph-mon[25094]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:22.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:22 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:53:23.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:53:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:53:23.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:53:24.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:24 smithi163 ceph-mon[25094]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:24 smithi163 ceph-mon[29791]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:24 smithi186 ceph-mon[25944]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:26 smithi163 ceph-mon[25094]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:26 smithi163 ceph-mon[29791]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:26 smithi186 ceph-mon[25944]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:27.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:27 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:27.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:27 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:27 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:28 smithi163 ceph-mon[25094]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:28.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:28 smithi163 ceph-mon[29791]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:28 smithi186 ceph-mon[25944]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:28.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:53:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:53:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:53:30.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:30 smithi163 ceph-mon[29791]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:30.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:30 smithi163 ceph-mon[25094]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:30 smithi186 ceph-mon[25944]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:32.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:32 smithi186 ceph-mon[25944]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:32.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:53:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:53:32] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.33.4" 2024-06-18T02:53:32.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:32 smithi163 ceph-mon[29791]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:32.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:32 smithi163 ceph-mon[25094]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:33.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:53:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:53:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:53:34.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:34 smithi163 ceph-mon[29791]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:34.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:34 smithi163 ceph-mon[25094]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:34 smithi186 ceph-mon[25944]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:36.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:36 smithi163 ceph-mon[29791]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:36 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:53:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:36 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:53:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:36 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:36 smithi163 ceph-mon[25094]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:36 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:53:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:36 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:53:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:36 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:36 smithi186 ceph-mon[25944]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:36 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:53:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:36 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:53:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:36 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:37.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:37 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:53:37.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:37 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:37 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:53:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:37 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:37 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:53:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:37 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:38.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:38 smithi163 ceph-mon[25094]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:38 smithi163 ceph-mon[29791]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:38 smithi186 ceph-mon[25944]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:38.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:53:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:53:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:53:40.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:40 smithi163 ceph-mon[29791]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:40.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:40 smithi163 ceph-mon[25094]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:40 smithi186 ceph-mon[25944]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:42.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:42 smithi186 ceph-mon[25944]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:42.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:53:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:53:42] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-06-18T02:53:42.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:42 smithi163 ceph-mon[29791]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:42.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:42 smithi163 ceph-mon[25094]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:43.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:53:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:53:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:53:44.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:44 smithi163 ceph-mon[29791]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:44 smithi163 ceph-mon[25094]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:44 smithi186 ceph-mon[25944]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:46.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:46 smithi163 ceph-mon[29791]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:46 smithi163 ceph-mon[25094]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:46 smithi186 ceph-mon[25944]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:47.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:47 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:47.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:47 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:47 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:48.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:48 smithi163 ceph-mon[29791]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:48.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:48 smithi163 ceph-mon[25094]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:48 smithi186 ceph-mon[25944]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:48.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:53:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:53:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:53:50.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:50 smithi163 ceph-mon[29791]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:50.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:50 smithi163 ceph-mon[25094]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:50.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:50 smithi186 ceph-mon[25944]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:52.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:52 smithi186 ceph-mon[25944]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:52.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:52 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:53:52.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:53:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:53:52] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-06-18T02:53:52.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:52 smithi163 ceph-mon[29791]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:52.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:52 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:53:52.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:52 smithi163 ceph-mon[25094]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:52.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:52 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:53:53.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:53:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:53:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:53:54.575 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-06-18T02:53:54.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:54 smithi163 ceph-mon[29791]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:54.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:54 smithi163 ceph-mon[25094]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:54 smithi186 ceph-mon[25944]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:53:56.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:56 smithi163 ceph-mon[25094]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:56.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:56 smithi163 ceph-mon[29791]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:56 smithi186 ceph-mon[25944]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:56.926 INFO:teuthology.orchestra.run.smithi163.stdout:Scheduled to redeploy mgr.y on host 'smithi163' 2024-06-18T02:53:57.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1953021549' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-18T02:53:57.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: from='client.15183 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:53:57.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:57.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: Schedule redeploy daemon mgr.y 2024-06-18T02:53:57.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:57.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:57.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:53:57.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:53:57.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:57.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:53:57.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:53:57.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T02:53:57.485 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:53:57.485 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: Deploying daemon mgr.y on smithi163 2024-06-18T02:53:57.485 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:57.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1953021549' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-18T02:53:57.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: from='client.15183 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:53:57.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:57.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: Schedule redeploy daemon mgr.y 2024-06-18T02:53:57.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:57.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:57.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:53:57.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:53:57.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:57.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:53:57.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:53:57.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T02:53:57.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:53:57.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: Deploying daemon mgr.y on smithi163 2024-06-18T02:53:57.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:57 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:57.507 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps --refresh' 2024-06-18T02:53:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1953021549' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-18T02:53:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: from='client.15183 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:53:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: Schedule redeploy daemon mgr.y 2024-06-18T02:53:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:53:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:53:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:53:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:53:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T02:53:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T02:53:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:53:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: Deploying daemon mgr.y on smithi163 2024-06-18T02:53:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:57 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:53:58.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:53:58 smithi163 ceph-mon[25094]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:53:58 smithi163 ceph-mon[29791]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:53:58 smithi186 ceph-mon[25944]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:53:58.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:53:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:53:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:53:59.857 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T02:53:59.858 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (2m) 85s ago 10m 14.1M - 0.23.0 ba2b418f427c 5e109a483ef4 2024-06-18T02:53:59.858 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (2m) 84s ago 10m 43.1M - 8.3.5 dad864ee21e9 a3fd9ade8374 2024-06-18T02:53:59.858 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 running (2m) 85s ago 9m 52.5M - 3.5 e1d6a67b021e f950ce5b4369 2024-06-18T02:53:59.858 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283 running (5m) 84s ago 15m 502M - 19.0.0-3124-g3781e3a5 af93c036e016 d7bd16a633ff 2024-06-18T02:53:59.858 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:9283 running (16m) 85s ago 16m 411M - 17.2.0 e1d6a67b021e 256bee9cc14e 2024-06-18T02:53:59.858 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (16m) 85s ago 16m 59.6M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T02:53:59.858 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (15m) 84s ago 15m 46.4M 2048M 17.2.0 e1d6a67b021e a3618f734a9f 2024-06-18T02:53:59.858 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (15m) 85s ago 15m 46.0M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T02:53:59.858 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (2m) 85s ago 11m 15.2M - 1.3.1 1dbe0e931976 8d0aecfcef8a 2024-06-18T02:53:59.858 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (2m) 84s ago 11m 10.0M - 1.3.1 1dbe0e931976 0592fede9474 2024-06-18T02:53:59.858 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (14m) 85s ago 14m 53.0M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T02:53:59.858 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (14m) 85s ago 14m 53.3M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T02:53:59.859 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (14m) 85s ago 14m 49.4M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T02:53:59.859 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (13m) 85s ago 13m 51.3M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T02:53:59.859 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (13m) 84s ago 13m 57.5M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T02:53:59.859 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (12m) 84s ago 12m 49.6M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T02:53:59.859 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (12m) 84s ago 12m 47.8M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T02:53:59.859 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (12m) 84s ago 12m 51.3M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T02:53:59.859 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (2m) 84s ago 11m 60.1M - 2.33.4 514e6a882f6e 8512fab6ffb8 2024-06-18T02:53:59.859 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (10m) 85s ago 9m 75.3M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T02:53:59.859 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (9m) 84s ago 9m 75.5M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T02:54:00.662 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:00 smithi163 ceph-mon[25094]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:00.662 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:00 smithi163 ceph-mon[25094]: from='client.15189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:54:00.663 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:00 smithi163 ceph-mon[29791]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:00.663 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:00 smithi163 ceph-mon[29791]: from='client.15189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:54:00.691 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'sleep 180' 2024-06-18T02:54:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:00 smithi186 ceph-mon[25944]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:00 smithi186 ceph-mon[25944]: from='client.15189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:54:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:02 smithi186 ceph-mon[25944]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:02.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:54:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:54:02] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-06-18T02:54:02.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:02 smithi163 ceph-mon[25094]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:02.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:02 smithi163 ceph-mon[29791]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:03.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:54:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:54:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:54:04.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:04 smithi163 ceph-mon[29791]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:04.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:04 smithi163 ceph-mon[25094]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:04.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:04 smithi186 ceph-mon[25944]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:06.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:06 smithi163 ceph-mon[29791]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:06.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:06 smithi163 ceph-mon[25094]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:06 smithi186 ceph-mon[25944]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:07.704 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:07 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:54:07.704 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:07 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:07.705 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:07 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:54:07.705 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:07 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:07 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:54:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:07 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:08 smithi163 ceph-mon[29791]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:08.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:08 smithi163 ceph-mon[25094]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:08 smithi186 ceph-mon[25944]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:08.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:54:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:54:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:54:10.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:10 smithi163 ceph-mon[29791]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-18T02:54:10.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:10 smithi163 ceph-mon[25094]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-18T02:54:10.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:10 smithi186 ceph-mon[25944]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-18T02:54:12.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:12 smithi186 ceph-mon[25944]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-06-18T02:54:12.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:54:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:54:12] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-06-18T02:54:12.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:12 smithi163 ceph-mon[29791]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-06-18T02:54:12.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:12 smithi163 ceph-mon[25094]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-06-18T02:54:13.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:54:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:54:13.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:54:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:14 smithi163 ceph-mon[29791]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-18T02:54:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:14 smithi163 ceph-mon[25094]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-18T02:54:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:14 smithi186 ceph-mon[25944]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-18T02:54:16.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:16 smithi163 ceph-mon[29791]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:16 smithi163 ceph-mon[25094]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:16 smithi186 ceph-mon[25944]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:17.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:17 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:17.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:17 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:17 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:18.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:18 smithi163 ceph-mon[29791]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:18.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:18 smithi163 ceph-mon[25094]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:18 smithi186 ceph-mon[25944]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:18.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:54:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:54:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:54:20.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:20 smithi163 ceph-mon[25094]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:20.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:20 smithi163 ceph-mon[29791]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:20.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:20 smithi186 ceph-mon[25944]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-06-18T02:54:22.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:22 smithi186 ceph-mon[25944]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:22.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:22 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:54:22.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:54:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:54:22] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-06-18T02:54:22.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:22 smithi163 ceph-mon[29791]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:22.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:22 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:54:22.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:22 smithi163 ceph-mon[25094]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:22.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:22 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:54:23.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:54:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:54:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:54:24.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:24 smithi163 ceph-mon[29791]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:24.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:24 smithi163 ceph-mon[25094]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:24 smithi186 ceph-mon[25944]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:26.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:26 smithi163 ceph-mon[29791]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:26.694 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:26 smithi163 ceph-mon[25094]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:26 smithi186 ceph-mon[25944]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:27.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:26 smithi163 systemd[1]: Stopping Ceph mgr.y for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:54:27.423 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:27 smithi163 podman[62582]: 2024-06-18 02:54:27.179154043 +0000 UTC m=+0.155009529 container died 256bee9cc14e72f3662285d078cbdbbfaa6fcb471514c76c3b7cc50d6b9a4ee7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, architecture=x86_64, version=8, vcs-type=git, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container) 2024-06-18T02:54:27.699 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:27 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:27.699 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:27 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:27.699 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:27 smithi163 podman[62582]: 2024-06-18 02:54:27.514944716 +0000 UTC m=+0.490800200 container cleanup 256bee9cc14e72f3662285d078cbdbbfaa6fcb471514c76c3b7cc50d6b9a4ee7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y, distribution-scope=public, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_CLEAN=True, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, architecture=x86_64) 2024-06-18T02:54:27.700 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:27 smithi163 bash[62582]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y 2024-06-18T02:54:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:27 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:27.960 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:27 smithi163 podman[62596]: 2024-06-18 02:54:27.699756584 +0000 UTC m=+0.520127064 container remove 256bee9cc14e72f3662285d078cbdbbfaa6fcb471514c76c3b7cc50d6b9a4ee7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y, architecture=x86_64, version=8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.buildah.version=1.19.8, release=754, distribution-scope=public, com.redhat.component=centos-stream-container, io.openshift.expose-services=, vcs-type=git, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, vendor=Red Hat, Inc.) 2024-06-18T02:54:27.960 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:27 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-06-18T02:54:27.960 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:27 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-06-18T02:54:27.960 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:27 smithi163 systemd[1]: Stopped Ceph mgr.y for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:54:27.960 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:27 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.y.service: Consumed 1min 1.909s CPU time. 2024-06-18T02:54:28.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:28 smithi163 systemd[1]: Starting Ceph mgr.y for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:54:28.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:28 smithi163 ceph-mon[25094]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:28.620 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:28 smithi163 ceph-mon[29791]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:28.620 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:28 smithi163 podman[62725]: 2024-06-18 02:54:28.351009305 +0000 UTC m=+0.107616909 container create 1477473a7a9cb55c495620bbe881eb7047dd72a45de5415d5ba4754f9968bd69 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-3781e3a, org.label-schema.build-date=20240612, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-06-18T02:54:28.620 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:28 smithi163 podman[62725]: 2024-06-18 02:54:28.260712586 +0000 UTC m=+0.017320157 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T02:54:28.620 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:28 smithi163 podman[62725]: 2024-06-18 02:54:28.545016575 +0000 UTC m=+0.301624142 container init 1477473a7a9cb55c495620bbe881eb7047dd72a45de5415d5ba4754f9968bd69 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y, GIT_BRANCH=HEAD, org.label-schema.build-date=20240612, GIT_CLEAN=True, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True) 2024-06-18T02:54:28.620 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:28 smithi163 podman[62725]: 2024-06-18 02:54:28.548148747 +0000 UTC m=+0.304756306 container start 1477473a7a9cb55c495620bbe881eb7047dd72a45de5415d5ba4754f9968bd69 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, RELEASE=squid-3781e3a, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-18T02:54:28.620 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:28 smithi163 ceph-mgr[62744]: -- 172.21.15.163:0/2057650585 <== mon.1 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5645b99630e0 con 0x5645b9940800 2024-06-18T02:54:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:28 smithi186 ceph-mon[25944]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:28.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:54:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:54:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:54:28.912 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:28 smithi163 bash[62725]: 1477473a7a9cb55c495620bbe881eb7047dd72a45de5415d5ba4754f9968bd69 2024-06-18T02:54:28.912 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:28 smithi163 systemd[1]: Started Ceph mgr.y for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:54:28.912 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:28.678+0000 7f0db7f77040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T02:54:28.912 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:28.839+0000 7f0db7f77040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:54:28.912 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:28.911+0000 7f0db7f77040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T02:54:29.513 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:29.190+0000 7f0db7f77040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T02:54:29.513 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:29.252+0000 7f0db7f77040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T02:54:29.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:29 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:29.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:29 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:29.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:29 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:29.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:29 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:29 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:29 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:29 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:29 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:30.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:29 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:30.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:29 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:30.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:29 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:30.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:29 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:30.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:30.261+0000 7f0db7f77040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T02:54:30.543 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:30.388+0000 7f0db7f77040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T02:54:30.543 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:30.455+0000 7f0db7f77040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T02:54:31.001 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:30 smithi163 ceph-mon[29791]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:31.001 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:30 smithi163 ceph-mon[25094]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:31.002 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:30.702+0000 7f0db7f77040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T02:54:31.002 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:30.767+0000 7f0db7f77040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T02:54:31.002 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:30.848+0000 7f0db7f77040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T02:54:31.002 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:31 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:31.001+0000 7f0db7f77040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:54:31.023 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:30 smithi186 ceph-mon[25944]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:31.783 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:31 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:31.434+0000 7f0db7f77040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T02:54:31.784 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:31 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:31.783+0000 7f0db7f77040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T02:54:32.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:31 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:31.852+0000 7f0db7f77040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T02:54:32.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:32 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:32.010+0000 7f0db7f77040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-18T02:54:32.403 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:32 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:32.335+0000 7f0db7f77040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T02:54:32.403 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:32 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:32.403+0000 7f0db7f77040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T02:54:32.556 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:54:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:54:32] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-06-18T02:54:32.703 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:32 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:32.614+0000 7f0db7f77040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T02:54:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:32 smithi186 ceph-mon[25944]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:32.998 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:32 smithi163 ceph-mon[29791]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:32.999 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:32 smithi163 ceph-mon[25094]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:32.999 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:32 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:32.774+0000 7f0db7f77040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T02:54:32.999 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:32 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:32.846+0000 7f0db7f77040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T02:54:32.999 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:32 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:32.936+0000 7f0db7f77040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T02:54:32.999 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:32 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:32.998+0000 7f0db7f77040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T02:54:33.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:54:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:54:33.818 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:33.563+0000 7f0db7f77040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T02:54:33.818 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:33.728+0000 7f0db7f77040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T02:54:33.818 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:02:54:33] ENGINE Bus STARTING 2024-06-18T02:54:33.819 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: CherryPy Checker: 2024-06-18T02:54:33.819 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: The Application mounted at '' has an empty config. 2024-06-18T02:54:33.819 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:54:34.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:02:54:33] ENGINE Serving on http://:::9283 2024-06-18T02:54:34.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:02:54:33] ENGINE Bus STARTED 2024-06-18T02:54:34.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:54:34.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:54:34.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[29791]: Standby manager daemon y restarted 2024-06-18T02:54:34.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[29791]: Standby manager daemon y started 2024-06-18T02:54:34.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-18T02:54:34.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:54:34.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-18T02:54:34.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:54:34.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:54:34.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:54:34.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[25094]: Standby manager daemon y restarted 2024-06-18T02:54:34.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[25094]: Standby manager daemon y started 2024-06-18T02:54:34.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-18T02:54:34.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:54:34.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-18T02:54:34.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:33 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:54:34.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:54:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:54:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:54:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-mon[25944]: Standby manager daemon y restarted 2024-06-18T02:54:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-mon[25944]: Standby manager daemon y started 2024-06-18T02:54:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-18T02:54:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:54:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-18T02:54:34.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:33 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:54:35.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:34 smithi163 ceph-mon[29791]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:35.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:34 smithi163 ceph-mon[29791]: mgrmap e26: x(active, since 3m), standbys: y 2024-06-18T02:54:35.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:34 smithi163 ceph-mon[25094]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:35.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:34 smithi163 ceph-mon[25094]: mgrmap e26: x(active, since 3m), standbys: y 2024-06-18T02:54:35.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:34 smithi186 ceph-mon[25944]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:35.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:34 smithi186 ceph-mon[25944]: mgrmap e26: x(active, since 3m), standbys: y 2024-06-18T02:54:36.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:36 smithi163 ceph-mon[29791]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:36 smithi163 ceph-mon[25094]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:36 smithi186 ceph-mon[25944]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:37.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:37 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:54:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:37 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:54:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:37 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:54:38.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:38 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:38 smithi163 ceph-mon[29791]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:38 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:38 smithi163 ceph-mon[25094]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:38 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:38 smithi186 ceph-mon[25944]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:38.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:54:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:54:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:54:40.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:40 smithi163 ceph-mon[29791]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:40.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:40 smithi163 ceph-mon[25094]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:40 smithi186 ceph-mon[25944]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:42.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:42 smithi186 ceph-mon[25944]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:42.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:54:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:54:42] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-18T02:54:42.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:42 smithi163 ceph-mon[29791]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:42.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:42 smithi163 ceph-mon[25094]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:43.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:54:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:54:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:54:44.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:44 smithi163 ceph-mon[29791]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:44 smithi163 ceph-mon[25094]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:44 smithi186 ceph-mon[25944]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:46.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:46 smithi163 ceph-mon[29791]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:46 smithi163 ceph-mon[25094]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:46 smithi186 ceph-mon[25944]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:48.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:48 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:48.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:48 smithi163 ceph-mon[29791]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:48.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:48 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:48.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:48 smithi163 ceph-mon[25094]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:48 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:48 smithi186 ceph-mon[25944]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:48.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:54:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:54:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:54:50.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:50 smithi163 ceph-mon[25094]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:50.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:50 smithi163 ceph-mon[29791]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:50.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:50 smithi186 ceph-mon[25944]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:52.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:52 smithi186 ceph-mon[25944]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:52.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:52 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:54:52.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:54:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:54:52] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-18T02:54:52.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:52 smithi163 ceph-mon[29791]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:52.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:52 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:54:52.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:52 smithi163 ceph-mon[25094]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:52.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:52 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:54:53.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:54:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:54:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:54:54.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:54 smithi163 ceph-mon[29791]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:54.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:54 smithi163 ceph-mon[25094]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:54 smithi186 ceph-mon[25944]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:54:56.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:56 smithi163 ceph-mon[25094]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:56.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:56 smithi163 ceph-mon[29791]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:56 smithi186 ceph-mon[25944]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:58.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:58 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:58.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:54:58 smithi163 ceph-mon[29791]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:58.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:58 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:58.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:54:58 smithi163 ceph-mon[25094]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:58 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:54:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:54:58 smithi186 ceph-mon[25944]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:54:58.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:54:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:54:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:55:00.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:00 smithi163 ceph-mon[29791]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:00.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:00 smithi163 ceph-mon[25094]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:00 smithi186 ceph-mon[25944]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:02 smithi186 ceph-mon[25944]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:02.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:55:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:55:02] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-18T02:55:02.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:02 smithi163 ceph-mon[29791]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:02.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:02 smithi163 ceph-mon[25094]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:03.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:55:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:55:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:55:04.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:04 smithi163 ceph-mon[29791]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:04.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:04 smithi163 ceph-mon[25094]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:04.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:04 smithi186 ceph-mon[25944]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:06.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:06 smithi163 ceph-mon[29791]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:06.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:06 smithi163 ceph-mon[25094]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:06 smithi186 ceph-mon[25944]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:07.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:07 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:55:07.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:07 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:55:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:07 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:55:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:08 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:08 smithi163 ceph-mon[29791]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:08.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:08 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:08.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:08 smithi163 ceph-mon[25094]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:08 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:08 smithi186 ceph-mon[25944]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:08.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:55:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:55:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:55:10.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:10 smithi163 ceph-mon[29791]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:10.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:10 smithi163 ceph-mon[25094]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:10.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:10 smithi186 ceph-mon[25944]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:12.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:12 smithi186 ceph-mon[25944]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:12.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:55:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:55:12] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-06-18T02:55:12.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:12 smithi163 ceph-mon[29791]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:12.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:12 smithi163 ceph-mon[25094]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:13.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:55:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:55:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:55:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:14 smithi186 ceph-mon[25944]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:15.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:14 smithi163 ceph-mon[29791]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:15.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:14 smithi163 ceph-mon[25094]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:16.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:16 smithi163 ceph-mon[29791]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:16 smithi163 ceph-mon[25094]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:16 smithi186 ceph-mon[25944]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:18.540 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:18 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:18.540 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:18 smithi186 ceph-mon[25944]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:18.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:18 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:18.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:18 smithi163 ceph-mon[29791]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:18.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:18 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:18.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:18 smithi163 ceph-mon[25094]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:18.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:55:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:55:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:55:20.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:20 smithi163 ceph-mon[29791]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:20 smithi163 ceph-mon[25094]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:20.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:20 smithi186 ceph-mon[25944]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:22.544 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:55:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:55:22] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-06-18T02:55:22.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:22 smithi186 ceph-mon[25944]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:22.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:22 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:55:23.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:22 smithi163 ceph-mon[29791]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:23.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:22 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:55:23.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:22 smithi163 ceph-mon[25094]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:23.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:22 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:55:23.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:55:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:55:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:55:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:24 smithi186 ceph-mon[25944]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:24.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:24 smithi163 ceph-mon[29791]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:24.845 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:24 smithi163 ceph-mon[25094]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:26 smithi163 ceph-mon[25094]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:26 smithi163 ceph-mon[29791]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:26 smithi186 ceph-mon[25944]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:28.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:28 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:28.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:28 smithi163 ceph-mon[29791]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:28 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:28 smithi163 ceph-mon[25094]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:28 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:28 smithi186 ceph-mon[25944]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:28.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:55:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:55:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:55:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:30 smithi186 ceph-mon[25944]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:31.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:30 smithi163 ceph-mon[29791]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:31.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:30 smithi163 ceph-mon[25094]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:32.548 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:55:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:55:32] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-06-18T02:55:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:32 smithi186 ceph-mon[25944]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:33.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:32 smithi163 ceph-mon[29791]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:33.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:32 smithi163 ceph-mon[25094]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:33.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:55:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:55:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:55:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:34 smithi186 ceph-mon[25944]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:35.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:34 smithi163 ceph-mon[29791]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:35.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:34 smithi163 ceph-mon[25094]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:36.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:36 smithi163 ceph-mon[29791]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:36 smithi163 ceph-mon[25094]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:36 smithi186 ceph-mon[25944]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:37.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:37 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:55:37.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:37 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:55:37.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:37 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:55:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:37 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:55:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:37 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:55:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:37 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:55:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:37 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:55:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:37 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:55:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:37 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:55:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:38 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:38 smithi163 ceph-mon[29791]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:38 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:55:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:38 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:55:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:38 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:55:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:38 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:38 smithi163 ceph-mon[25094]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:38 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:55:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:38 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:55:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:38 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:55:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:38 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:38 smithi186 ceph-mon[25944]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:38 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:55:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:38 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:55:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:38 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:55:38.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:55:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:55:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:55:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:40 smithi186 ceph-mon[25944]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:41.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:40 smithi163 ceph-mon[29791]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:41.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:40 smithi163 ceph-mon[25094]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:42.549 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:55:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:55:42] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-06-18T02:55:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:42 smithi186 ceph-mon[25944]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:43.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:42 smithi163 ceph-mon[29791]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:42 smithi163 ceph-mon[25094]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:43.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:55:43 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:55:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:55:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:44 smithi186 ceph-mon[25944]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:45.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:44 smithi163 ceph-mon[29791]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:44 smithi163 ceph-mon[25094]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:46.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:46 smithi163 ceph-mon[29791]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:46 smithi163 ceph-mon[25094]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:46 smithi186 ceph-mon[25944]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:48 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:48 smithi186 ceph-mon[25944]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:48.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:55:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:55:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:55:49.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:48 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:49.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:48 smithi163 ceph-mon[29791]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:49.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:48 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:49.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:48 smithi163 ceph-mon[25094]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:50.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:50 smithi186 ceph-mon[25944]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:51.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:50 smithi163 ceph-mon[29791]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:51.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:50 smithi163 ceph-mon[25094]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:52.553 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:55:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:55:52] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-06-18T02:55:52.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:52 smithi186 ceph-mon[25944]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:52.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:52 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:55:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:52 smithi163 ceph-mon[29791]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:52 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:55:53.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:52 smithi163 ceph-mon[25094]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:53.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:52 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:55:53.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:55:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:55:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:55:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:54 smithi186 ceph-mon[25944]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:54.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:54 smithi163 ceph-mon[29791]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:54.845 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:54 smithi163 ceph-mon[25094]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:55:56.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:56 smithi163 ceph-mon[25094]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:56.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:56 smithi163 ceph-mon[29791]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:56 smithi186 ceph-mon[25944]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:58 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:55:58 smithi186 ceph-mon[25944]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:58.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:55:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:55:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi186:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:55:59.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:58 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:59.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:55:58 smithi163 ceph-mon[25094]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:55:59.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:58 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:55:59.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:55:58 smithi163 ceph-mon[29791]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:00 smithi186 ceph-mon[25944]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:01.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:00 smithi163 ceph-mon[29791]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:01.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:00 smithi163 ceph-mon[25094]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:02.559 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:56:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:56:02] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-06-18T02:56:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:02 smithi186 ceph-mon[25944]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:03.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:02 smithi163 ceph-mon[29791]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:02 smithi163 ceph-mon[25094]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:03.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:56:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:56:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi163\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"929affee-2d1b-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi163\", job=\"node\", machine=\"x86_64\", nodename=\"smithi163\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-18T02:56:04.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:04 smithi186 ceph-mon[25944]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:05.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:04 smithi163 ceph-mon[29791]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:04 smithi163 ceph-mon[25094]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:06.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:06 smithi163 ceph-mon[25094]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:06.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:06 smithi163 ceph-mon[29791]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:06 smithi186 ceph-mon[25944]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:07 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:56:07.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:07 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:56:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:07 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:56:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:08 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:08 smithi163 ceph-mon[29791]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:08.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:08 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:08.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:08 smithi163 ceph-mon[25094]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:08 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:08 smithi186 ceph-mon[25944]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:10.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:10 smithi186 ceph-mon[25944]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:11.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:10 smithi163 ceph-mon[25094]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:11.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:10 smithi163 ceph-mon[29791]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:12.560 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:56:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:56:12] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-06-18T02:56:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:12 smithi186 ceph-mon[25944]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:13.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:12 smithi163 ceph-mon[29791]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:13.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:12 smithi163 ceph-mon[25094]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:14 smithi186 ceph-mon[25944]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:15.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:14 smithi163 ceph-mon[29791]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:15.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:14 smithi163 ceph-mon[25094]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:16.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:16 smithi163 ceph-mon[29791]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:16 smithi163 ceph-mon[25094]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:16 smithi186 ceph-mon[25944]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:18 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:18 smithi186 ceph-mon[25944]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:19.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:18 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:19.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:18 smithi163 ceph-mon[25094]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:19.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:18 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:19.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:18 smithi163 ceph-mon[29791]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:20.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:20 smithi186 ceph-mon[25944]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:21.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:20 smithi163 ceph-mon[29791]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:21.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:20 smithi163 ceph-mon[25094]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:22.565 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:56:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:56:22] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-06-18T02:56:22.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:22 smithi186 ceph-mon[25944]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:22.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:22 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:56:23.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:22 smithi163 ceph-mon[25094]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:23.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:22 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:56:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:22 smithi163 ceph-mon[29791]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:22 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:56:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:24 smithi186 ceph-mon[25944]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:24.845 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:24 smithi163 ceph-mon[25094]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:24.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:24 smithi163 ceph-mon[29791]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:26 smithi163 ceph-mon[25094]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-18T02:56:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:26 smithi163 ceph-mon[29791]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-18T02:56:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:26 smithi186 ceph-mon[25944]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-18T02:56:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:28 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:28 smithi186 ceph-mon[25944]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-18T02:56:29.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:28 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:29.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:28 smithi163 ceph-mon[25094]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-18T02:56:29.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:28 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:29.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:28 smithi163 ceph-mon[29791]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-18T02:56:31.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:30 smithi163 ceph-mon[29791]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:31.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:30 smithi163 ceph-mon[25094]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:31.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:30 smithi186 ceph-mon[25944]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:32.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:56:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:56:32] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-06-18T02:56:33.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:32 smithi163 ceph-mon[25094]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-18T02:56:33.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:32 smithi163 ceph-mon[29791]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-18T02:56:33.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:32 smithi186 ceph-mon[25944]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-18T02:56:35.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:34 smithi163 ceph-mon[29791]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:35.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:34 smithi163 ceph-mon[25094]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:35.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:34 smithi186 ceph-mon[25944]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:36.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:36 smithi163 ceph-mon[25094]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:36 smithi163 ceph-mon[29791]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:36 smithi186 ceph-mon[25944]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:37.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:37 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:56:37.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:37 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:56:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:37 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:56:38.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:38 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:38.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:38 smithi163 ceph-mon[29791]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:38 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:56:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:38 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:56:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:38 smithi163 ceph-mon[29791]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:56:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:38 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:38 smithi163 ceph-mon[25094]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:38 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:56:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:38 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:56:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:38 smithi163 ceph-mon[25094]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:56:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:38 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:38 smithi186 ceph-mon[25944]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:38 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:56:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:38 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:56:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:38 smithi186 ceph-mon[25944]: from='mgr.24701 ' entity='mgr.x' 2024-06-18T02:56:41.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:40 smithi163 ceph-mon[29791]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:41.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:40 smithi163 ceph-mon[25094]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:40 smithi186 ceph-mon[25944]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:42.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:56:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:56:42] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-06-18T02:56:43.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:42 smithi163 ceph-mon[29791]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:42 smithi163 ceph-mon[25094]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:43.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:42 smithi186 ceph-mon[25944]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:45.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:44 smithi163 ceph-mon[25094]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:45.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:44 smithi163 ceph-mon[29791]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:44 smithi186 ceph-mon[25944]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:46.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:46 smithi163 ceph-mon[29791]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:46.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:46 smithi163 ceph-mon[25094]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:46 smithi186 ceph-mon[25944]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:49.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:48 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:49.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:48 smithi163 ceph-mon[29791]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:49.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:48 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:49.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:48 smithi163 ceph-mon[25094]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:49.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:48 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:49.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:48 smithi186 ceph-mon[25944]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:51.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:50 smithi163 ceph-mon[29791]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:51.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:50 smithi163 ceph-mon[25094]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:51.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:50 smithi186 ceph-mon[25944]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:52.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:56:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:56:52] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-06-18T02:56:53.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:52 smithi163 ceph-mon[29791]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:53.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:52 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:56:53.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:52 smithi163 ceph-mon[25094]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:53.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:52 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:56:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:52 smithi186 ceph-mon[25944]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:52 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:56:54.844 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:54 smithi163 ceph-mon[29791]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:54.844 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:54 smithi163 ceph-mon[25094]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:55.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:54 smithi186 ceph-mon[25944]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:56:56.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:56 smithi163 ceph-mon[25094]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:56.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:56 smithi163 ceph-mon[29791]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:56 smithi186 ceph-mon[25944]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:59.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:58 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:59.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:56:58 smithi163 ceph-mon[25094]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:59.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:58 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:59.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:56:58 smithi163 ceph-mon[29791]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:56:59.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:58 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:56:59.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:56:58 smithi186 ceph-mon[25944]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:01.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:00 smithi163 ceph-mon[25094]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:57:01.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:00 smithi163 ceph-mon[29791]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:57:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:00 smithi186 ceph-mon[25944]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:57:02.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:57:02] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-06-18T02:57:03.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:02 smithi163 ceph-mon[25094]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:03.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:02 smithi163 ceph-mon[29791]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:02 smithi186 ceph-mon[25944]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:04.372 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps' 2024-06-18T02:57:04.682 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:04 smithi163 ceph-mon[29791]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:57:04.682 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:04 smithi163 ceph-mon[25094]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:57:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:04 smithi186 ceph-mon[25944]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:57:06.319 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T02:57:06.319 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (6m) 89s ago 13m 21.4M - 0.23.0 ba2b418f427c 5e109a483ef4 2024-06-18T02:57:06.319 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (5m) 2m ago 13m 43.7M - 8.3.5 dad864ee21e9 a3fd9ade8374 2024-06-18T02:57:06.319 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 running (5m) 89s ago 12m 55.5M - 3.5 e1d6a67b021e f950ce5b4369 2024-06-18T02:57:06.319 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283 running (9m) 2m ago 18m 504M - 19.0.0-3124-g3781e3a5 af93c036e016 d7bd16a633ff 2024-06-18T02:57:06.319 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (2m) 89s ago 19m 438M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T02:57:06.319 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (20m) 89s ago 20m 64.0M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (18m) 2m ago 18m 48.9M 2048M 17.2.0 e1d6a67b021e a3618f734a9f 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (18m) 89s ago 18m 42.1M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (6m) 89s ago 14m 16.0M - 1.3.1 1dbe0e931976 8d0aecfcef8a 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (5m) 2m ago 14m 17.2M - 1.3.1 1dbe0e931976 0592fede9474 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (17m) 89s ago 17m 53.9M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (17m) 89s ago 17m 54.4M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (17m) 89s ago 17m 49.9M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (16m) 89s ago 16m 52.5M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (16m) 2m ago 16m 58.3M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (16m) 2m ago 16m 50.5M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (15m) 2m ago 15m 49.1M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (15m) 2m ago 15m 52.8M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (5m) 2m ago 14m 71.0M - 2.33.4 514e6a882f6e 8512fab6ffb8 2024-06-18T02:57:06.320 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (13m) 89s ago 13m 76.1M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T02:57:06.321 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (13m) 2m ago 13m 76.0M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T02:57:06.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:06 smithi163 ceph-mon[29791]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:06.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:06 smithi163 ceph-mon[25094]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:06 smithi186 ceph-mon[25944]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:06.954 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions' 2024-06-18T02:57:07.786 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:07 smithi163 ceph-mon[29791]: from='client.15201 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:57:07.786 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:07 smithi163 ceph-mon[29791]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:57:07.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:07 smithi163 ceph-mon[25094]: from='client.15201 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:57:07.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:07 smithi163 ceph-mon[25094]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:57:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:07 smithi186 ceph-mon[25944]: from='client.15201 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T02:57:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:07 smithi186 ceph-mon[25944]: from='mgr.24701 172.21.15.186:0/59983611' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:57:08.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:08 smithi163 ceph-mon[29791]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:57:08.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:08 smithi163 ceph-mon[29791]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:08.787 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T02:57:08.787 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T02:57:08.787 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-18T02:57:08.787 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:57:08.787 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T02:57:08.787 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T02:57:08.788 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:57:08.788 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T02:57:08.788 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-18T02:57:08.788 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:57:08.788 INFO:teuthology.orchestra.run.smithi163.stdout: "mds": {}, 2024-06-18T02:57:08.788 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T02:57:08.788 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T02:57:08.788 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T02:57:08.788 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T02:57:08.788 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-18T02:57:08.788 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T02:57:08.788 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T02:57:08.788 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T02:57:08.789 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:08 smithi163 ceph-mon[25094]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:57:08.789 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:08 smithi163 ceph-mon[25094]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:08 smithi186 ceph-mon[25944]: from='client.25057 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:57:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:08 smithi186 ceph-mon[25944]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:09.421 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph health detail' 2024-06-18T02:57:09.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:09 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2056296739' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T02:57:09.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:09 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2056296739' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T02:57:09.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:09 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2056296739' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T02:57:10.592 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:10 smithi163 ceph-mon[29791]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:57:10.592 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:10 smithi163 ceph-mon[25094]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:57:10.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:10 smithi186 ceph-mon[25944]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:57:11.350 INFO:teuthology.orchestra.run.smithi163.stdout:HEALTH_OK 2024-06-18T02:57:11.656 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:11 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2115367567' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T02:57:11.657 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:11 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2115367567' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T02:57:11.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:11 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2115367567' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T02:57:11.989 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph -s' 2024-06-18T02:57:12.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:12 smithi186 ceph-mon[25944]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:12.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:57:12] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-06-18T02:57:12.634 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:12 smithi163 ceph-mon[29791]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:12.634 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:12 smithi163 ceph-mon[25094]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:14.080 INFO:teuthology.orchestra.run.smithi163.stdout: cluster: 2024-06-18T02:57:14.080 INFO:teuthology.orchestra.run.smithi163.stdout: id: 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T02:57:14.080 INFO:teuthology.orchestra.run.smithi163.stdout: health: HEALTH_OK 2024-06-18T02:57:14.080 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:57:14.080 INFO:teuthology.orchestra.run.smithi163.stdout: services: 2024-06-18T02:57:14.080 INFO:teuthology.orchestra.run.smithi163.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-06-18T02:57:14.080 INFO:teuthology.orchestra.run.smithi163.stdout: mgr: x(active, since 6m), standbys: y 2024-06-18T02:57:14.081 INFO:teuthology.orchestra.run.smithi163.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-06-18T02:57:14.081 INFO:teuthology.orchestra.run.smithi163.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-06-18T02:57:14.081 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:57:14.081 INFO:teuthology.orchestra.run.smithi163.stdout: data: 2024-06-18T02:57:14.081 INFO:teuthology.orchestra.run.smithi163.stdout: pools: 6 pools, 161 pgs 2024-06-18T02:57:14.081 INFO:teuthology.orchestra.run.smithi163.stdout: objects: 241 objects, 585 KiB 2024-06-18T02:57:14.081 INFO:teuthology.orchestra.run.smithi163.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:14.081 INFO:teuthology.orchestra.run.smithi163.stdout: pgs: 161 active+clean 2024-06-18T02:57:14.081 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:57:14.081 INFO:teuthology.orchestra.run.smithi163.stdout: io: 2024-06-18T02:57:14.081 INFO:teuthology.orchestra.run.smithi163.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-06-18T02:57:14.081 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T02:57:14.738 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:14 smithi163 ceph-mon[25094]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:57:14.738 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:14 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/508303457' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-18T02:57:14.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:14 smithi163 ceph-mon[29791]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:57:14.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:14 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/508303457' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-18T02:57:14.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph mgr fail' 2024-06-18T02:57:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:14 smithi186 ceph-mon[25944]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:57:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:14 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/508303457' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-18T02:57:16.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:16 smithi163 ceph-mon[29791]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:16 smithi163 ceph-mon[25094]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:16 smithi186 ceph-mon[25944]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:17.753 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:17 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1809572433' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T02:57:17.753 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:17 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T02:57:17.753 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:17 smithi163 ceph-mon[29791]: osdmap e103: 8 total, 8 up, 8 in 2024-06-18T02:57:17.753 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:17 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1809572433' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T02:57:17.754 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:17 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T02:57:17.754 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:17 smithi163 ceph-mon[25094]: osdmap e103: 8 total, 8 up, 8 in 2024-06-18T02:57:17.754 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:17 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:02:57:17] ENGINE Bus STOPPING 2024-06-18T02:57:17.754 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:17 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:02:57:17] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-18T02:57:17.754 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:17 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:02:57:17] ENGINE Bus STOPPED 2024-06-18T02:57:17.754 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:17 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:02:57:17] ENGINE Bus STARTING 2024-06-18T02:57:17.755 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:17 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:02:57:17] ENGINE Serving on http://:::9283 2024-06-18T02:57:17.791 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:17 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1809572433' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T02:57:17.792 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:17 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T02:57:17.792 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:17 smithi186 ceph-mon[25944]: osdmap e103: 8 total, 8 up, 8 in 2024-06-18T02:57:17.792 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:17.484+0000 7f95e7ecb640 -1 mgr handle_mgr_map I was active but no longer am 2024-06-18T02:57:17.792 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ignoring --setuser ceph since I am not root 2024-06-18T02:57:17.792 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ignoring --setgroup ceph since I am not root 2024-06-18T02:57:17.792 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:17 smithi186 ceph-mgr[45515]: -- 172.21.15.186:0/482510761 <== mon.2 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556bca8ad0e0 con 0x556bca88a800 2024-06-18T02:57:17.792 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:17.640+0000 7f87f2fc1040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T02:57:17.792 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:17.791+0000 7f87f2fc1040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:57:18.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:17 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:02:57:17] ENGINE Bus STARTED 2024-06-18T02:57:18.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:17.863+0000 7f87f2fc1040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T02:57:18.166 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'sleep 180' 2024-06-18T02:57:18.490 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:18.116+0000 7f87f2fc1040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T02:57:18.490 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:18.181+0000 7f87f2fc1040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T02:57:18.778 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-18T02:57:18.778 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: mgrmap e27: y(active, starting, since 0.99439s) 2024-06-18T02:57:18.779 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:57:18.779 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:57:18.779 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:57:18.779 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T02:57:18.779 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:57:18.779 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:57:18.779 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:57:18.779 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:57:18.779 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:57:18.779 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:57:18.779 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:57:18.779 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:57:18.779 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T02:57:18.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T02:57:18.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T02:57:18.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: Manager daemon y is now available 2024-06-18T02:57:18.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:57:18.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:57:18.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:57:18.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:57:18.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:57:18.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-18T02:57:18.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: mgrmap e27: y(active, starting, since 0.99439s) 2024-06-18T02:57:18.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:57:18.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:57:18.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: Manager daemon y is now available 2024-06-18T02:57:18.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:57:18.782 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:57:18.782 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:57:18.782 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:57:18.782 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:18 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: mgrmap e27: y(active, starting, since 0.99439s) 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T02:57:18.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T02:57:18.789 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T02:57:18.789 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T02:57:18.789 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T02:57:18.789 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: Manager daemon y is now available 2024-06-18T02:57:18.789 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:57:18.789 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:57:18.789 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T02:57:18.789 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:57:18.789 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:18 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T02:57:19.340 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:57:19.061+0000 7f0d0fa65640 -1 log_channel(cephadm) log [ERR] : [18/Jun/2024:02:57:19] ENGINE Error in HTTPServer.serve 2024-06-18T02:57:19.341 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Traceback (most recent call last): 2024-06-18T02:57:19.341 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T02:57:19.341 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: self._connections.run(self.expiration_interval) 2024-06-18T02:57:19.341 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T02:57:19.341 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: self._run(expiration_interval) 2024-06-18T02:57:19.341 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T02:57:19.341 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T02:57:19.341 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T02:57:19.341 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T02:57:19.341 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T02:57:19.341 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: s = self.context.wrap_socket( 2024-06-18T02:57:19.341 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T02:57:19.341 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: return self.sslsocket_class._create( 2024-06-18T02:57:19.342 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T02:57:19.342 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: self.do_handshake() 2024-06-18T02:57:19.342 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T02:57:19.342 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: self._sslobj.do_handshake() 2024-06-18T02:57:19.342 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T02:57:19.342 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:57:19.494 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:19 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:19.193+0000 7f87f2fc1040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T02:57:19.494 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:19 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:19.314+0000 7f87f2fc1040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T02:57:19.494 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:19 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:19.377+0000 7f87f2fc1040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T02:57:19.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-mon[29791]: mgrmap e28: y(active, since 1.99702s) 2024-06-18T02:57:19.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-mon[29791]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-mon[25094]: mgrmap e28: y(active, since 1.99702s) 2024-06-18T02:57:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:19 smithi163 ceph-mon[25094]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:19.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:19 smithi186 ceph-mon[25944]: mgrmap e28: y(active, since 1.99702s) 2024-06-18T02:57:19.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:19 smithi186 ceph-mon[25944]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:19.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:19 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:19.605+0000 7f87f2fc1040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T02:57:19.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:19 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:19.673+0000 7f87f2fc1040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T02:57:19.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:19 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:19.743+0000 7f87f2fc1040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T02:57:20.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:19 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:19.910+0000 7f87f2fc1040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T02:57:20.657 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:20 smithi163 ceph-mon[29791]: [18/Jun/2024:02:57:18] ENGINE Bus STARTING 2024-06-18T02:57:20.658 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:20 smithi163 ceph-mon[29791]: [18/Jun/2024:02:57:18] ENGINE Serving on http://172.21.15.163:8765 2024-06-18T02:57:20.658 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:20 smithi163 ceph-mon[29791]: [18/Jun/2024:02:57:19] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T02:57:20.658 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:20 smithi163 ceph-mon[29791]: [18/Jun/2024:02:57:19] ENGINE Bus STARTED 2024-06-18T02:57:20.658 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:20 smithi163 ceph-mon[29791]: [18/Jun/2024:02:57:19] ENGINE Error in HTTPServer.serve 2024-06-18T02:57:20.658 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:57:20.658 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T02:57:20.658 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._connections.run(self.expiration_interval) 2024-06-18T02:57:20.658 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T02:57:20.658 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._run(expiration_interval) 2024-06-18T02:57:20.658 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T02:57:20.658 INFO:journalctl@ceph.mon.c.smithi163.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T02:57:20.658 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T02:57:20.659 INFO:journalctl@ceph.mon.c.smithi163.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T02:57:20.659 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T02:57:20.659 INFO:journalctl@ceph.mon.c.smithi163.stdout: s = self.context.wrap_socket( 2024-06-18T02:57:20.659 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T02:57:20.659 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.sslsocket_class._create( 2024-06-18T02:57:20.659 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T02:57:20.659 INFO:journalctl@ceph.mon.c.smithi163.stdout: self.do_handshake() 2024-06-18T02:57:20.659 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T02:57:20.659 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._sslobj.do_handshake() 2024-06-18T02:57:20.659 INFO:journalctl@ceph.mon.c.smithi163.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T02:57:20.659 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:20 smithi163 ceph-mon[29791]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:20.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:20 smithi163 ceph-mon[25094]: [18/Jun/2024:02:57:18] ENGINE Bus STARTING 2024-06-18T02:57:20.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:20 smithi163 ceph-mon[25094]: [18/Jun/2024:02:57:18] ENGINE Serving on http://172.21.15.163:8765 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:20 smithi163 ceph-mon[25094]: [18/Jun/2024:02:57:19] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:20 smithi163 ceph-mon[25094]: [18/Jun/2024:02:57:19] ENGINE Bus STARTED 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:20 smithi163 ceph-mon[25094]: [18/Jun/2024:02:57:19] ENGINE Error in HTTPServer.serve 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._connections.run(self.expiration_interval) 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._run(expiration_interval) 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout: s = self.context.wrap_socket( 2024-06-18T02:57:20.660 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T02:57:20.661 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.sslsocket_class._create( 2024-06-18T02:57:20.661 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T02:57:20.661 INFO:journalctl@ceph.mon.a.smithi163.stdout: self.do_handshake() 2024-06-18T02:57:20.661 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T02:57:20.661 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._sslobj.do_handshake() 2024-06-18T02:57:20.661 INFO:journalctl@ceph.mon.a.smithi163.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T02:57:20.661 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:20 smithi163 ceph-mon[25094]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:20.705 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:20 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:20.369+0000 7f87f2fc1040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T02:57:20.706 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:20 smithi186 ceph-mon[25944]: [18/Jun/2024:02:57:18] ENGINE Bus STARTING 2024-06-18T02:57:20.706 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:20 smithi186 ceph-mon[25944]: [18/Jun/2024:02:57:18] ENGINE Serving on http://172.21.15.163:8765 2024-06-18T02:57:20.706 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:20 smithi186 ceph-mon[25944]: [18/Jun/2024:02:57:19] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T02:57:20.706 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:20 smithi186 ceph-mon[25944]: [18/Jun/2024:02:57:19] ENGINE Bus STARTED 2024-06-18T02:57:20.706 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:20 smithi186 ceph-mon[25944]: [18/Jun/2024:02:57:19] ENGINE Error in HTTPServer.serve 2024-06-18T02:57:20.706 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T02:57:20.706 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T02:57:20.706 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._connections.run(self.expiration_interval) 2024-06-18T02:57:20.706 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T02:57:20.706 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._run(expiration_interval) 2024-06-18T02:57:20.706 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T02:57:20.707 INFO:journalctl@ceph.mon.b.smithi186.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T02:57:20.707 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T02:57:20.707 INFO:journalctl@ceph.mon.b.smithi186.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T02:57:20.707 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T02:57:20.707 INFO:journalctl@ceph.mon.b.smithi186.stdout: s = self.context.wrap_socket( 2024-06-18T02:57:20.707 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T02:57:20.707 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.sslsocket_class._create( 2024-06-18T02:57:20.707 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T02:57:20.707 INFO:journalctl@ceph.mon.b.smithi186.stdout: self.do_handshake() 2024-06-18T02:57:20.707 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T02:57:20.707 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._sslobj.do_handshake() 2024-06-18T02:57:20.707 INFO:journalctl@ceph.mon.b.smithi186.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T02:57:20.707 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:20 smithi186 ceph-mon[25944]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:21.062 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:20 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:20.704+0000 7f87f2fc1040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T02:57:21.062 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:20 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:20.774+0000 7f87f2fc1040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T02:57:21.062 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:20 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:20.939+0000 7f87f2fc1040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-18T02:57:21.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:21.264+0000 7f87f2fc1040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T02:57:21.690 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:21 smithi186 ceph-mon[25944]: mgrmap e29: y(active, since 4s) 2024-06-18T02:57:21.690 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:21.327+0000 7f87f2fc1040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T02:57:21.690 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:21.521+0000 7f87f2fc1040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T02:57:21.690 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:21.688+0000 7f87f2fc1040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T02:57:21.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:21 smithi163 ceph-mon[29791]: mgrmap e29: y(active, since 4s) 2024-06-18T02:57:21.695 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:21 smithi163 ceph-mon[25094]: mgrmap e29: y(active, since 4s) 2024-06-18T02:57:21.945 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:21.754+0000 7f87f2fc1040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T02:57:21.945 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:21.846+0000 7f87f2fc1040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T02:57:21.945 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:21.909+0000 7f87f2fc1040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T02:57:22.763 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:22 smithi186 ceph-mon[25944]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:22.763 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:22 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:22.764 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:22 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:22.764 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:22.480+0000 7f87f2fc1040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T02:57:22.764 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:22.655+0000 7f87f2fc1040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T02:57:22.764 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:57:22] ENGINE Bus STARTING 2024-06-18T02:57:22.764 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: CherryPy Checker: 2024-06-18T02:57:22.764 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: The Application mounted at '' has an empty config. 2024-06-18T02:57:22.764 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T02:57:22.764 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:57:22] ENGINE Serving on http://:::9283 2024-06-18T02:57:22.764 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 02:57:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:02:57:22] ENGINE Bus STARTED 2024-06-18T02:57:22.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:22 smithi163 ceph-mon[29791]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:22.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:22 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:22.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:22 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:22.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:22 smithi163 ceph-mon[25094]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:22.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:22 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:22.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:22 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:23.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[29791]: Standby manager daemon x started 2024-06-18T02:57:23.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T02:57:23.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:57:23.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T02:57:23.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:57:23.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:23.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:23.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:23.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[25094]: Standby manager daemon x started 2024-06-18T02:57:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T02:57:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:57:23.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T02:57:23.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:57:23.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:23.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:23.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:23.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:23 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:23 smithi186 ceph-mon[25944]: Standby manager daemon x started 2024-06-18T02:57:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:23 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T02:57:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:23 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T02:57:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:23 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T02:57:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:23 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T02:57:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:23 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:23 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:23 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:23.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:23 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:24.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[29791]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:24.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[29791]: mgrmap e30: y(active, since 7s), standbys: x 2024-06-18T02:57:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T02:57:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:57:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[25094]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[25094]: mgrmap e30: y(active, since 7s), standbys: x 2024-06-18T02:57:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T02:57:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:24 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:57:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:24 smithi186 ceph-mon[25944]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:24 smithi186 ceph-mon[25944]: mgrmap e30: y(active, since 7s), standbys: x 2024-06-18T02:57:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:24 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T02:57:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:24 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:24 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:24 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:24 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:57:25.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T02:57:25.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T02:57:25.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T02:57:25.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T02:57:25.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:57:25.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:57:25.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T02:57:25.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T02:57:25.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:57:25.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:57:25.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:25.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T02:57:25.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T02:57:25.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T02:57:25.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T02:57:25.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:57:25.797 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:57:25.797 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T02:57:25.797 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T02:57:25.797 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.797 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.797 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.797 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.797 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:57:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:57:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:25 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T02:57:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T02:57:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T02:57:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T02:57:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:57:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T02:57:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T02:57:25.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T02:57:25.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:25.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:57:25.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:57:25.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:25 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:26 smithi163 ceph-mon[25094]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T02:57:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:26 smithi163 ceph-mon[25094]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:57:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:26 smithi163 ceph-mon[25094]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-18T02:57:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:26 smithi163 ceph-mon[29791]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T02:57:26.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:26 smithi163 ceph-mon[29791]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:57:26.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:26 smithi163 ceph-mon[29791]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-18T02:57:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:26 smithi186 ceph-mon[25944]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T02:57:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:26 smithi186 ceph-mon[25944]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:57:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:26 smithi186 ceph-mon[25944]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-18T02:57:27.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T02:57:27.586+0000 7f0d48cd4640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:57:27.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Creating ceph-iscsi config... 2024-06-18T02:57:27.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:57:27.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:57:27.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:57:27.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:27.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:27.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Traceback (most recent call last): 2024-06-18T02:57:27.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:57:27.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: return _run_code(code, main_globals, None, 2024-06-18T02:57:27.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:57:27.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: exec(code, run_globals) 2024-06-18T02:57:27.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:57:27.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:57:27.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:57:27.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:57:27.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:57:27.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:57:27.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:57:27.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:27.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:27.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Traceback (most recent call last): 2024-06-18T02:57:27.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T02:57:27.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T02:57:27.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T02:57:27.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: return self.wait_async( 2024-06-18T02:57:27.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T02:57:27.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: return self.event_loop.get_result(coro, timeout) 2024-06-18T02:57:27.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T02:57:27.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: return future.result(timeout) 2024-06-18T02:57:27.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T02:57:27.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: return self.__get_result() 2024-06-18T02:57:27.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T02:57:27.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: raise self._exception 2024-06-18T02:57:27.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T02:57:27.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: out, err, code = await self._run_cephadm( 2024-06-18T02:57:27.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T02:57:27.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: raise OrchestratorError( 2024-06-18T02:57:27.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:57:27.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Creating ceph-iscsi config... 2024-06-18T02:57:27.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:57:27.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Traceback (most recent call last): 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: return _run_code(code, main_globals, None, 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: exec(code, run_globals) 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:57:27.797 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:57:27.798 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:57:27.798 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:27.798 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:28.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:28 smithi163 ceph-mon[25094]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-18T02:57:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:28 smithi163 ceph-mon[25094]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:57:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T02:57:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:57:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:57:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:57:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:57:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:57:28.794 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T02:57:28.794 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:57:28.794 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T02:57:28.794 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:57:28.794 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:57:28.794 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:57:28.794 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:57:28.794 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:57:28.794 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:57:28.794 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:57:28.794 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:28.795 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:28.795 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:57:28.795 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T02:57:28.795 INFO:journalctl@ceph.mon.a.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T02:57:28.795 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T02:57:28.795 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.wait_async( 2024-06-18T02:57:28.795 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T02:57:28.795 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T02:57:28.795 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T02:57:28.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:28 smithi163 ceph-mon[29791]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-18T02:57:28.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:28 smithi163 ceph-mon[29791]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:57:28.796 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T02:57:28.796 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:57:28.796 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:57:28.796 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:57:28.796 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:28.796 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:28.796 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:57:28.796 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:57:28.796 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T02:57:28.796 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:57:28.796 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T02:57:28.796 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.wait_async( 2024-06-18T02:57:28.797 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: return future.result(timeout) 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.__get_result() 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise self._exception 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise OrchestratorError( 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T02:57:28.798 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:57:28.799 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:57:28.799 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:57:28.799 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:28.799 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:28.799 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:57:28.799 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:57:28.799 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T02:57:28.799 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:57:28.799 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T02:57:28.799 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:57:28.799 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:57:28.799 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:57:28.799 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:57:28.800 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:57:28.800 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:57:28.800 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:57:28.800 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:28.800 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:28.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:28 smithi163 ceph-mon[29791]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T02:57:28.800 INFO:journalctl@ceph.mon.a.smithi163.stdout: return future.result(timeout) 2024-06-18T02:57:28.800 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T02:57:28.800 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.__get_result() 2024-06-18T02:57:28.800 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise self._exception 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise OrchestratorError( 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:57:28.801 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T02:57:28.802 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:57:28.802 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T02:57:28.802 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:57:28.802 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:57:28.802 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:57:28.802 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:57:28.802 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:57:28.802 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:57:28.802 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:57:28.802 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:28.802 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:28.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:28 smithi163 ceph-mon[25094]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T02:57:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:28 smithi186 ceph-mon[25944]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-18T02:57:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:28 smithi186 ceph-mon[25944]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:57:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T02:57:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:57:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:57:28.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:57:28.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:28.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:28.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T02:57:28.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:57:28.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T02:57:28.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:57:28.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T02:57:28.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:57:28.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:57:28.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:57:28.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:57:28.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.wait_async( 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: return future.result(timeout) 2024-06-18T02:57:28.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.__get_result() 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise self._exception 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: out, err, code = await self._run_cephadm( 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise OrchestratorError( 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:28.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:28.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T02:57:28.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T02:57:28.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T02:57:28.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T02:57:28.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T02:57:28.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T02:57:28.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T02:57:28.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T02:57:28.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T02:57:28.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T02:57:28.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T02:57:28.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T02:57:28.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T02:57:28.823 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T02:57:28.823 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:28 smithi186 ceph-mon[25944]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T02:57:29.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:29 smithi163 ceph-mon[29791]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T02:57:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:29 smithi163 ceph-mon[25094]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T02:57:29.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:29 smithi186 ceph-mon[25944]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T02:57:30.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:30 smithi163 ceph-mon[29791]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T02:57:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:30 smithi163 ceph-mon[25094]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T02:57:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:30 smithi186 ceph-mon[25944]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T02:57:31.315 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:30 smithi186 systemd[1]: Stopping Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:57:31.315 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:30 smithi186 bash[54306]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-18T02:57:31.315 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:57:31.033Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-18T02:57:31.315 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:57:31.033Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-06-18T02:57:31.315 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:57:31.033Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-06-18T02:57:31.316 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:57:31.033Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-06-18T02:57:31.316 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:57:31.033Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-06-18T02:57:31.316 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:57:31.033Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-06-18T02:57:31.316 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:57:31.033Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-18T02:57:31.316 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:57:31.033Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-06-18T02:57:31.316 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:57:31.033Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-06-18T02:57:31.316 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:57:31.034Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-06-18T02:57:31.316 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:57:31.034Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-06-18T02:57:31.316 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[49077]: ts=2024-06-18T02:57:31.034Z caller=main.go:1066 level=info msg="See you next time!" 2024-06-18T02:57:31.316 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 podman[54317]: 2024-06-18 02:57:31.04385287 +0000 UTC m=+0.114103455 container died 8512fab6ffb82ede667814a1201132ba8f169fc95482e7a200a35286c445efc4 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:57:31.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 podman[54317]: 2024-06-18 02:57:31.382759593 +0000 UTC m=+0.453010174 container cleanup 8512fab6ffb82ede667814a1201132ba8f169fc95482e7a200a35286c445efc4 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:57:31.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 bash[54317]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a 2024-06-18T02:57:31.883 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 podman[54331]: 2024-06-18 02:57:31.592414442 +0000 UTC m=+0.549654400 container remove 8512fab6ffb82ede667814a1201132ba8f169fc95482e7a200a35286c445efc4 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:57:31.884 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 bash[54344]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-18T02:57:31.884 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-06-18T02:57:31.884 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 systemd[1]: Stopped Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:57:31.884 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a.service: Consumed 2.133s CPU time. 2024-06-18T02:57:31.884 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 systemd[1]: Starting Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T02:57:32.153 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 podman[54408]: 2024-06-18 02:57:31.787816918 +0000 UTC m=+0.018698595 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-06-18T02:57:32.153 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:31 smithi186 podman[54408]: 2024-06-18 02:57:31.910323476 +0000 UTC m=+0.141205149 container create eba156538173038e749e5fb353fe2ef8ca9543671cbbad420d68246701c77c5b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:57:32.153 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 podman[54408]: 2024-06-18 02:57:32.153746095 +0000 UTC m=+0.384627773 container init eba156538173038e749e5fb353fe2ef8ca9543671cbbad420d68246701c77c5b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:57:32.406 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 podman[54408]: 2024-06-18 02:57:32.156579657 +0000 UTC m=+0.387461331 container start eba156538173038e749e5fb353fe2ef8ca9543671cbbad420d68246701c77c5b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T02:57:32.406 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.180Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-06-18T02:57:32.406 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.180Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-06-18T02:57:32.406 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.180Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-06-18T02:57:32.406 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.180Z caller=main.go:518 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi186 (none))" 2024-06-18T02:57:32.406 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.180Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-18T02:57:32.406 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.180Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-18T02:57:32.406 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.182Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-06-18T02:57:32.406 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.182Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-06-18T02:57:32.407 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.183Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-06-18T02:57:32.407 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.185Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-18T02:57:32.407 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.185Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.113µs 2024-06-18T02:57:32.407 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.185Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-18T02:57:32.407 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 bash[54408]: eba156538173038e749e5fb353fe2ef8ca9543671cbbad420d68246701c77c5b 2024-06-18T02:57:32.407 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 systemd[1]: Started Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T02:57:32.407 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.405Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-06-18T02:57:32.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.645Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-06-18T02:57:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-mon[25944]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:57:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:57:33.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:32 smithi163 ceph-mon[29791]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:57:33.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:32 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:33.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:32 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:33.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:32 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:57:33.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:32 smithi163 ceph-mon[25094]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:57:33.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:32 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:33.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:32 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:33.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:32 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:57:33.166 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.884Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-06-18T02:57:33.166 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.884Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-06-18T02:57:33.166 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.884Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.716µs wal_replay_duration=698.975711ms total_replay_duration=699.025911ms 2024-06-18T02:57:33.166 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.889Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-06-18T02:57:33.166 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.889Z caller=main.go:947 level=info msg="TSDB started" 2024-06-18T02:57:33.166 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.889Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-18T02:57:33.166 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.908Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.118247ms db_storage=699ns remote_storage=1.14µs web_handler=300ns query_engine=766ns scrape=419.285µs scrape_sd=116.662µs notify=18.59µs notify_sd=16.974µs rules=18.079158ms 2024-06-18T02:57:33.166 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 02:57:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T02:57:32.908Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-06-18T02:57:33.782 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:33 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:57:33.782 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:33 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:57:34.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:33 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:57:34.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:33 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:57:34.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:33 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T02:57:34.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:33 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:57:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:34 smithi186 ceph-mon[25944]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:57:35.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:34 smithi163 ceph-mon[29791]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:57:35.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:34 smithi163 ceph-mon[25094]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:57:36.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:36 smithi163 ceph-mon[29791]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:57:36.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:36 smithi163 ceph-mon[25094]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:57:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:36 smithi186 ceph-mon[25944]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T02:57:37.647 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:37.647 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:37.647 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:37.647 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:57:37.647 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:37.647 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:57:37.647 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:57:37.647 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:37.647 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:37.648 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:37.648 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:37.648 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:57:37.648 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:37.648 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:57:37.648 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:57:37.648 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:37 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:38.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:37 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:38.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:37 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:38.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:37 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:38.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:37 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:57:38.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:37 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:38.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:37 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:57:38.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:37 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T02:57:38.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:37 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:38 smithi163 ceph-mon[25094]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T02:57:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:38 smithi163 ceph-mon[25094]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:57:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:38 smithi163 ceph-mon[25094]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-18T02:57:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:38 smithi163 ceph-mon[29791]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T02:57:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:38 smithi163 ceph-mon[29791]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:57:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:38 smithi163 ceph-mon[29791]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-18T02:57:39.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:38 smithi186 ceph-mon[25944]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T02:57:39.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:38 smithi186 ceph-mon[25944]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T02:57:39.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:38 smithi186 ceph-mon[25944]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-18T02:57:40.918 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:40 smithi163 ceph-mon[29791]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-18T02:57:40.918 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:40 smithi163 ceph-mon[25094]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-18T02:57:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:40 smithi186 ceph-mon[25944]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-18T02:57:42.752 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:42.752 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:42.752 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:57:42.752 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:57:42.752 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:57:42.752 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:57:42.752 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[25094]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:42.752 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3000676919' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T02:57:42.753 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:42.753 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:42.753 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:57:42.753 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:57:42.753 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:57:42.753 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:57:42.753 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[29791]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:42.753 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:42 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3000676919' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T02:57:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:42 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:42 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:42 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:57:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:42 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-18T02:57:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:42 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:57:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:42 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-18T02:57:42.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:42 smithi186 ceph-mon[25944]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:42.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:42 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3000676919' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T02:57:44.240 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:57:43] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.33.4" 2024-06-18T02:57:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:44 smithi186 ceph-mon[25944]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:44.951 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:44 smithi163 ceph-mon[29791]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:44.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:44 smithi163 ceph-mon[25094]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T02:57:46.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:46 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:46 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:46 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:46 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:57:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:46 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:46 smithi163 ceph-mon[25094]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-18T02:57:46.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:46 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:46.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:46 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:46.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:46 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:46.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:46 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:57:46.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:46 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:46.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:46 smithi163 ceph-mon[29791]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-18T02:57:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:46 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:46 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:46 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:57:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:46 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:57:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:46 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:57:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:46 smithi186 ceph-mon[25944]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-18T02:57:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:48 smithi186 ceph-mon[25944]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-18T02:57:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:48 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:57:49.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:48 smithi163 ceph-mon[25094]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-18T02:57:49.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:48 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:57:49.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:48 smithi163 ceph-mon[29791]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-18T02:57:49.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:48 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:57:50.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:50 smithi186 ceph-mon[25944]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-18T02:57:50.844 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:50 smithi163 ceph-mon[29791]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-18T02:57:50.844 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:50 smithi163 ceph-mon[25094]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-18T02:57:52.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:52 smithi186 ceph-mon[25944]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-18T02:57:52.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:52 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:57:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:52 smithi163 ceph-mon[29791]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-18T02:57:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:52 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:57:53.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:52 smithi163 ceph-mon[25094]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-18T02:57:53.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:52 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:57:54.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:57:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:57:53] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.33.4" 2024-06-18T02:57:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:54 smithi186 ceph-mon[25944]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-18T02:57:55.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:54 smithi163 ceph-mon[29791]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-18T02:57:55.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:54 smithi163 ceph-mon[25094]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-18T02:57:56.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:56 smithi163 ceph-mon[25094]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-18T02:57:56.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:56 smithi163 ceph-mon[29791]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-18T02:57:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:56 smithi186 ceph-mon[25944]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-18T02:57:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:57:58 smithi186 ceph-mon[25944]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:59.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:57:58 smithi163 ceph-mon[25094]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:57:59.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:57:58 smithi163 ceph-mon[29791]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:00 smithi186 ceph-mon[25944]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:01.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:00 smithi163 ceph-mon[29791]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:01.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:00 smithi163 ceph-mon[25094]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:02 smithi186 ceph-mon[25944]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:03.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:02 smithi163 ceph-mon[29791]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:03.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:02 smithi163 ceph-mon[25094]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:03.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:03 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:03.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:03 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:58:03.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:03 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:03.854 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:03 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:58:04.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:03 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:04.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:03 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:58:04.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:58:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:58:03] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.33.4" 2024-06-18T02:58:04.975 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:04 smithi186 ceph-mon[25944]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:05.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:04 smithi163 ceph-mon[29791]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:04 smithi163 ceph-mon[25094]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:06.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:06 smithi163 ceph-mon[25094]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:06 smithi163 ceph-mon[29791]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:06 smithi186 ceph-mon[25944]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:08 smithi186 ceph-mon[25944]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:09.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:08 smithi163 ceph-mon[25094]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:09.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:08 smithi163 ceph-mon[29791]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:10.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:10 smithi186 ceph-mon[25944]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:11.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:10 smithi163 ceph-mon[29791]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:11.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:10 smithi163 ceph-mon[25094]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:12 smithi186 ceph-mon[25944]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:13.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:12 smithi163 ceph-mon[29791]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:13.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:12 smithi163 ceph-mon[25094]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:13.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:13 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:13.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:13 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:13 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:14.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:58:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:58:13] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.33.4" 2024-06-18T02:58:15.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:14 smithi163 ceph-mon[29791]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:15.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:14 smithi163 ceph-mon[25094]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:14 smithi186 ceph-mon[25944]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:16.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:16 smithi163 ceph-mon[29791]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:16 smithi163 ceph-mon[25094]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:16 smithi186 ceph-mon[25944]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:18 smithi186 ceph-mon[25944]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:58:19.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:18 smithi163 ceph-mon[29791]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:19.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:58:19.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:18 smithi163 ceph-mon[25094]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:19.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:58:20.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:20 smithi186 ceph-mon[25944]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:21.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:20 smithi163 ceph-mon[29791]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:21.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:20 smithi163 ceph-mon[25094]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:23.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:22 smithi163 ceph-mon[29791]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:23.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:22 smithi163 ceph-mon[25094]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:23.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:22 smithi186 ceph-mon[25944]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:23.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:23 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:23.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:23 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:24.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:23 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:24.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:58:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:58:23] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-06-18T02:58:25.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:24 smithi163 ceph-mon[25094]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:25.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:24 smithi163 ceph-mon[29791]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:25.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:24 smithi186 ceph-mon[25944]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:26.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:26 smithi163 ceph-mon[29791]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:26 smithi163 ceph-mon[25094]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:26 smithi186 ceph-mon[25944]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:28 smithi186 ceph-mon[25944]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:29.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:28 smithi163 ceph-mon[29791]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:29.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:28 smithi163 ceph-mon[25094]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:30 smithi186 ceph-mon[25944]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:31.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:30 smithi163 ceph-mon[29791]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:31.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:30 smithi163 ceph-mon[25094]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:33.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:32 smithi163 ceph-mon[25094]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:33.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:32 smithi163 ceph-mon[29791]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:33.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:32 smithi186 ceph-mon[25944]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:33.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:33 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:33.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:33 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:58:33.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:33 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:33.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:33 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:58:33.969 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:33 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:33.969 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:33 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:58:34.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:58:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:58:33] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.33.4" 2024-06-18T02:58:35.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:34 smithi163 ceph-mon[29791]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:35.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:34 smithi163 ceph-mon[25094]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:35.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:34 smithi186 ceph-mon[25944]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:36.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:36 smithi163 ceph-mon[29791]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:36.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:36 smithi163 ceph-mon[25094]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:36 smithi186 ceph-mon[25944]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:39.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:38 smithi163 ceph-mon[29791]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:39.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:38 smithi163 ceph-mon[25094]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:39.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:38 smithi186 ceph-mon[25944]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:41.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:40 smithi163 ceph-mon[29791]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:41.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:40 smithi163 ceph-mon[25094]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:40 smithi186 ceph-mon[25944]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:43.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:42 smithi163 ceph-mon[29791]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:42 smithi163 ceph-mon[25094]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:43.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:42 smithi186 ceph-mon[25944]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:43.852 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:43 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:43.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:43 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:43 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:44.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:58:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:58:43] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.33.4" 2024-06-18T02:58:45.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:44 smithi163 ceph-mon[29791]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:44 smithi163 ceph-mon[25094]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:44 smithi186 ceph-mon[25944]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:46.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:46 smithi163 ceph-mon[29791]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:46.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:46 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:58:46.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:46 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:58:46.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:46 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:58:46.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:46 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:58:46.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:46 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:58:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:46 smithi163 ceph-mon[25094]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:46 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:58:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:46 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:58:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:46 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:58:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:46 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:58:46.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:46 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:58:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:46 smithi186 ceph-mon[25944]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:46 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:58:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:46 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:58:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:46 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:58:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:46 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:58:46.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:46 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:58:49.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:48 smithi163 ceph-mon[29791]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:49.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:48 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:58:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:48 smithi163 ceph-mon[25094]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:48 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:58:49.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:48 smithi186 ceph-mon[25944]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:49.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:48 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:58:51.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:50 smithi163 ceph-mon[29791]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:51.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:50 smithi163 ceph-mon[25094]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:51.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:50 smithi186 ceph-mon[25944]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:52 smithi163 ceph-mon[29791]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:53.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:52 smithi163 ceph-mon[25094]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:52 smithi186 ceph-mon[25944]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:53.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:53 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:53.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:53 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:54.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:53 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:58:54.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:58:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:58:53] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-06-18T02:58:55.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:54 smithi163 ceph-mon[29791]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:55.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:54 smithi163 ceph-mon[25094]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:55.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:54 smithi186 ceph-mon[25944]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:58:56.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:56 smithi163 ceph-mon[29791]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:56.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:56 smithi163 ceph-mon[25094]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:56 smithi186 ceph-mon[25944]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:59.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:58:58 smithi163 ceph-mon[29791]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:59.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:58:58 smithi163 ceph-mon[25094]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:58:59.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:58:58 smithi186 ceph-mon[25944]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:01.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:00 smithi163 ceph-mon[25094]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:01.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:00 smithi163 ceph-mon[29791]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:00 smithi186 ceph-mon[25944]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:03.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:02 smithi163 ceph-mon[25094]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:03.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:02 smithi163 ceph-mon[29791]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:02 smithi186 ceph-mon[25944]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:03.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:03 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:03.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:03 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:59:03.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:03 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:03.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:03 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:59:03.969 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:03 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:03.969 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:03 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:59:04.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:59:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:59:03] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-06-18T02:59:04.982 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:04 smithi186 ceph-mon[25944]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:05.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:04 smithi163 ceph-mon[25094]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:05.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:04 smithi163 ceph-mon[29791]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:06.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:06 smithi163 ceph-mon[25094]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:06.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:06 smithi163 ceph-mon[29791]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:06 smithi186 ceph-mon[25944]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:09.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:08 smithi163 ceph-mon[29791]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:09.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:08 smithi163 ceph-mon[25094]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:08 smithi186 ceph-mon[25944]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:11.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:10 smithi163 ceph-mon[29791]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:11.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:10 smithi163 ceph-mon[25094]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:11.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:10 smithi186 ceph-mon[25944]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:13.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:12 smithi163 ceph-mon[25094]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:13.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:12 smithi163 ceph-mon[29791]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:13.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:12 smithi186 ceph-mon[25944]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:13.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:13 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:13.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:13 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:13 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:14.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:59:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:59:13] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-06-18T02:59:15.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:14 smithi163 ceph-mon[25094]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:15.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:14 smithi163 ceph-mon[29791]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:14 smithi186 ceph-mon[25944]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:16.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:16 smithi163 ceph-mon[25094]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:16.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:16 smithi163 ceph-mon[29791]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:16 smithi186 ceph-mon[25944]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:19.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:18 smithi163 ceph-mon[29791]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:19.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:59:19.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:18 smithi163 ceph-mon[25094]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:19.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:59:19.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:18 smithi186 ceph-mon[25944]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:19.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:59:21.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:20 smithi163 ceph-mon[29791]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:21.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:20 smithi163 ceph-mon[25094]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:21.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:20 smithi186 ceph-mon[25944]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:23.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:22 smithi163 ceph-mon[25094]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:22 smithi163 ceph-mon[29791]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:23.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:22 smithi186 ceph-mon[25944]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:23.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:23 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:23.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:23 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:24.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:23 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:24.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:59:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:59:23] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-06-18T02:59:25.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:24 smithi163 ceph-mon[29791]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:25.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:24 smithi163 ceph-mon[25094]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:25.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:24 smithi186 ceph-mon[25944]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:26 smithi163 ceph-mon[25094]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:26 smithi163 ceph-mon[29791]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:26 smithi186 ceph-mon[25944]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:29.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:28 smithi163 ceph-mon[25094]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:29.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:28 smithi163 ceph-mon[29791]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:29.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:28 smithi186 ceph-mon[25944]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:31.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:30 smithi163 ceph-mon[29791]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:31.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:30 smithi163 ceph-mon[25094]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:31.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:30 smithi186 ceph-mon[25944]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:33.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:32 smithi163 ceph-mon[29791]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:33.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:32 smithi163 ceph-mon[25094]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:33.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:32 smithi186 ceph-mon[25944]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:33.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:33 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:33.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:33 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:59:33.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:33 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:33.854 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:33 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:59:33.969 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:33 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:33.970 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:33 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:59:34.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:59:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:59:33] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.33.4" 2024-06-18T02:59:35.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:34 smithi163 ceph-mon[29791]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:35.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:34 smithi163 ceph-mon[25094]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:35.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:34 smithi186 ceph-mon[25944]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:36 smithi163 ceph-mon[29791]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:36 smithi163 ceph-mon[25094]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:36 smithi186 ceph-mon[25944]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:39.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:38 smithi163 ceph-mon[29791]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:39.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:38 smithi163 ceph-mon[25094]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:39.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:38 smithi186 ceph-mon[25944]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:41.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:40 smithi163 ceph-mon[29791]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:41.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:40 smithi163 ceph-mon[25094]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:41.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:40 smithi186 ceph-mon[25944]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:43.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:42 smithi163 ceph-mon[25094]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:43.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:42 smithi163 ceph-mon[29791]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:43.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:42 smithi186 ceph-mon[25944]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:43.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:43 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:43.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:43 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:43 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:44.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:59:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:59:43] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.33.4" 2024-06-18T02:59:45.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:44 smithi163 ceph-mon[29791]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:44 smithi163 ceph-mon[25094]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:44 smithi186 ceph-mon[25944]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:46.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:46 smithi163 ceph-mon[25094]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:46.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:46 smithi163 ceph-mon[29791]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:46 smithi186 ceph-mon[25944]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:47.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:47 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:59:47.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:47 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:59:47.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:47 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:59:47.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:47 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:59:47.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:47 smithi163 ceph-mon[29791]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:59:47.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:47 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:59:47.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:47 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:59:47.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:47 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:59:47.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:47 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:59:47.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:47 smithi163 ceph-mon[25094]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:59:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:47 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:59:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:47 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:59:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:47 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T02:59:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:47 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T02:59:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:47 smithi186 ceph-mon[25944]: from='mgr.25099 ' entity='mgr.y' 2024-06-18T02:59:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:48 smithi186 ceph-mon[25944]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:48 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:59:49.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:48 smithi163 ceph-mon[29791]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:49.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:48 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:59:49.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:48 smithi163 ceph-mon[25094]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:49.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:48 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T02:59:50.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:50 smithi186 ceph-mon[25944]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:51.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:50 smithi163 ceph-mon[29791]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:51.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:50 smithi163 ceph-mon[25094]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:52.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:52 smithi186 ceph-mon[25944]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:52 smithi163 ceph-mon[29791]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:53.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:52 smithi163 ceph-mon[25094]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:53.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:53 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:53.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:53 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:53.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:53 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T02:59:54.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 02:59:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:02:59:53] "GET /metrics HTTP/1.1" 200 37917 "" "Prometheus/2.33.4" 2024-06-18T02:59:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:54 smithi186 ceph-mon[25944]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:55.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:54 smithi163 ceph-mon[29791]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:55.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:54 smithi163 ceph-mon[25094]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T02:59:56.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:56 smithi163 ceph-mon[25094]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:56.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:56 smithi163 ceph-mon[29791]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:56 smithi186 ceph-mon[25944]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:59.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 02:59:58 smithi163 ceph-mon[25094]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:59.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 02:59:58 smithi163 ceph-mon[29791]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T02:59:59.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 02:59:58 smithi186 ceph-mon[25944]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:01.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:00 smithi163 ceph-mon[29791]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:01.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:00 smithi163 ceph-mon[29791]: overall HEALTH_OK 2024-06-18T03:00:01.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:00 smithi163 ceph-mon[25094]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:01.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:00 smithi163 ceph-mon[25094]: overall HEALTH_OK 2024-06-18T03:00:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:00 smithi186 ceph-mon[25944]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:00 smithi186 ceph-mon[25944]: overall HEALTH_OK 2024-06-18T03:00:03.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:02 smithi163 ceph-mon[29791]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:03.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:02 smithi163 ceph-mon[25094]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:02 smithi186 ceph-mon[25944]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:03.970 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:03 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:03.970 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:03 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:04.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:03 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:04.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:03 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:04.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:03 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:04.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:03 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:04.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:00:03] "GET /metrics HTTP/1.1" 200 37917 "" "Prometheus/2.33.4" 2024-06-18T03:00:05.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:04 smithi163 ceph-mon[29791]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:04 smithi163 ceph-mon[25094]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:04 smithi186 ceph-mon[25944]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:06.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:06 smithi163 ceph-mon[29791]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:06.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:06 smithi163 ceph-mon[25094]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:06 smithi186 ceph-mon[25944]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:09.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:08 smithi163 ceph-mon[29791]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:09.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:08 smithi163 ceph-mon[25094]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:08 smithi186 ceph-mon[25944]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:11.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:10 smithi163 ceph-mon[25094]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:11.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:10 smithi163 ceph-mon[29791]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:11.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:10 smithi186 ceph-mon[25944]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:13.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:12 smithi163 ceph-mon[29791]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:13.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:12 smithi163 ceph-mon[25094]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:13.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:12 smithi186 ceph-mon[25944]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:14.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:13 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:14.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:13 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:14.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:00:13] "GET /metrics HTTP/1.1" 200 37917 "" "Prometheus/2.33.4" 2024-06-18T03:00:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:13 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:15.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:14 smithi163 ceph-mon[29791]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:15.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:14 smithi163 ceph-mon[25094]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:14 smithi186 ceph-mon[25944]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:16.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:16 smithi163 ceph-mon[29791]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:16 smithi163 ceph-mon[25094]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:16 smithi186 ceph-mon[25944]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:19.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:18 smithi163 ceph-mon[29791]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:19.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:18 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:19.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:18 smithi163 ceph-mon[25094]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:19.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:18 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:19.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:18 smithi186 ceph-mon[25944]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:19.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:18 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:20.991 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:20 smithi163 ceph-mon[29791]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:20.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:20 smithi163 ceph-mon[25094]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:21.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:20 smithi186 ceph-mon[25944]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:21.690 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps' 2024-06-18T03:00:22.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:22 smithi163 ceph-mon[25094]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:22.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:22 smithi163 ceph-mon[29791]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:23.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:22 smithi186 ceph-mon[25944]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:23.643 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:00:23.643 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (9m) 2m ago 16m 21.7M - 0.23.0 ba2b418f427c 5e109a483ef4 2024-06-18T03:00:23.643 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (9m) 2m ago 16m 52.7M - 8.3.5 dad864ee21e9 a3fd9ade8374 2024-06-18T03:00:23.643 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 running (2m) 2m ago 16m 73.7M - 3.5 e1d6a67b021e 2b1dde02e840 2024-06-18T03:00:23.643 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283 running (12m) 2m ago 21m 436M - 19.0.0-3124-g3781e3a5 af93c036e016 d7bd16a633ff 2024-06-18T03:00:23.643 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (5m) 2m ago 23m 492M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:00:23.643 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (23m) 2m ago 23m 69.8M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T03:00:23.643 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (21m) 2m ago 21m 43.8M 2048M 17.2.0 e1d6a67b021e a3618f734a9f 2024-06-18T03:00:23.644 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (22m) 2m ago 21m 45.4M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T03:00:23.644 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (9m) 2m ago 17m 15.9M - 1.3.1 1dbe0e931976 8d0aecfcef8a 2024-06-18T03:00:23.644 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (9m) 2m ago 17m 17.7M - 1.3.1 1dbe0e931976 0592fede9474 2024-06-18T03:00:23.644 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (21m) 2m ago 21m 54.6M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T03:00:23.644 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (20m) 2m ago 20m 54.5M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T03:00:23.644 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (20m) 2m ago 20m 50.2M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:00:23.644 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (20m) 2m ago 20m 53.0M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:00:23.644 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (19m) 2m ago 19m 58.8M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:00:23.644 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (19m) 2m ago 19m 51.2M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:00:23.644 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (18m) 2m ago 18m 49.8M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:00:23.644 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (18m) 2m ago 18m 53.4M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:00:23.645 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (2m) 2m ago 17m 37.7M - 2.33.4 514e6a882f6e eba156538173 2024-06-18T03:00:23.645 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (16m) 2m ago 16m 76.9M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:00:23.645 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (16m) 2m ago 16m 77.6M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:00:23.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:23 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:23.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:23 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:24.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:23 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:24.207 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:00:23] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.33.4" 2024-06-18T03:00:24.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions' 2024-06-18T03:00:24.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:24 smithi163 ceph-mon[25094]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:24.777 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:24 smithi163 ceph-mon[29791]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:25.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:24 smithi186 ceph-mon[25944]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:25.712 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:25 smithi163 ceph-mon[25094]: from='client.25189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:00:25.712 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:25 smithi163 ceph-mon[29791]: from='client.25189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:00:26.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:25 smithi186 ceph-mon[25944]: from='client.25189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:00:26.211 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:00:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:00:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-18T03:00:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "mds": {}, 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:00:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:00:26.213 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:00:26.821 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:26 smithi163 ceph-mon[29791]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:26.821 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:26 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3273729817' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:00:26.821 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:26 smithi163 ceph-mon[25094]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:26.821 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:26 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3273729817' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:00:26.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph -s' 2024-06-18T03:00:27.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:26 smithi186 ceph-mon[25944]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:27.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:26 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3273729817' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:00:28.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:28 smithi163 ceph-mon[29791]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:28 smithi163 ceph-mon[25094]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:28.961 INFO:teuthology.orchestra.run.smithi163.stdout: cluster: 2024-06-18T03:00:28.961 INFO:teuthology.orchestra.run.smithi163.stdout: id: 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T03:00:28.962 INFO:teuthology.orchestra.run.smithi163.stdout: health: HEALTH_OK 2024-06-18T03:00:28.962 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T03:00:28.962 INFO:teuthology.orchestra.run.smithi163.stdout: services: 2024-06-18T03:00:28.962 INFO:teuthology.orchestra.run.smithi163.stdout: mon: 3 daemons, quorum a,c,b (age 21m) 2024-06-18T03:00:28.962 INFO:teuthology.orchestra.run.smithi163.stdout: mgr: y(active, since 3m), standbys: x 2024-06-18T03:00:28.962 INFO:teuthology.orchestra.run.smithi163.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-06-18T03:00:28.962 INFO:teuthology.orchestra.run.smithi163.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-06-18T03:00:28.962 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T03:00:28.963 INFO:teuthology.orchestra.run.smithi163.stdout: data: 2024-06-18T03:00:28.963 INFO:teuthology.orchestra.run.smithi163.stdout: pools: 6 pools, 161 pgs 2024-06-18T03:00:28.963 INFO:teuthology.orchestra.run.smithi163.stdout: objects: 241 objects, 585 KiB 2024-06-18T03:00:28.963 INFO:teuthology.orchestra.run.smithi163.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:00:28.963 INFO:teuthology.orchestra.run.smithi163.stdout: pgs: 161 active+clean 2024-06-18T03:00:28.963 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T03:00:28.963 INFO:teuthology.orchestra.run.smithi163.stdout: io: 2024-06-18T03:00:28.963 INFO:teuthology.orchestra.run.smithi163.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-06-18T03:00:28.963 INFO:teuthology.orchestra.run.smithi163.stdout: 2024-06-18T03:00:29.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:28 smithi186 ceph-mon[25944]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:29.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph health detail' 2024-06-18T03:00:29.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:29 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1003499703' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-18T03:00:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:29 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1003499703' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-18T03:00:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:29 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1003499703' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-18T03:00:30.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:30 smithi163 ceph-mon[29791]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:30.897 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:30 smithi163 ceph-mon[25094]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:31.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:30 smithi186 ceph-mon[25944]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:31.576 INFO:teuthology.orchestra.run.smithi163.stdout:HEALTH_OK 2024-06-18T03:00:31.951 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:31 smithi163 ceph-mon[29791]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:31.951 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:31 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2528457061' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:00:31.951 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:31 smithi163 ceph-mon[25094]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:31.951 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:31 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2528457061' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:00:32.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:31 smithi186 ceph-mon[25944]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:32.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:31 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2528457061' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:00:32.241 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-06-18T03:00:32.904 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:32 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:32.905 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:32 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:33.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:32 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:33.878 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:33 smithi163 ceph-mon[29791]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:33.879 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:33 smithi163 ceph-mon[29791]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:33.879 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:33 smithi163 ceph-mon[25094]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:33.879 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:33 smithi163 ceph-mon[25094]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:33.879 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:00:33] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.33.4" 2024-06-18T03:00:33.969 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:33 smithi186 ceph-mon[25944]: from='mgr.25099 172.21.15.163:0/3576095211' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:33.969 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:33 smithi186 ceph-mon[25944]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:00:34.278 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:00:34.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:34 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3468100647' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:00:34.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:34 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3468100647' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:00:34.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph mgr fail' 2024-06-18T03:00:35.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:34 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3468100647' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:00:36.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:36 smithi163 ceph-mon[29791]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:36 smithi163 ceph-mon[25094]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:36 smithi186 ceph-mon[25944]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:00:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:37 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/546001753' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T03:00:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:37 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T03:00:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:37 smithi163 ceph-mon[25094]: osdmap e104: 8 total, 8 up, 8 in 2024-06-18T03:00:37.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:37.531+0000 7f0db2f08640 -1 mgr handle_mgr_map I was active but no longer am 2024-06-18T03:00:37.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ignoring --setuser ceph since I am not root 2024-06-18T03:00:37.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ignoring --setgroup ceph since I am not root 2024-06-18T03:00:37.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:37 smithi163 ceph-mgr[62744]: -- 172.21.15.163:0/3946279080 <== mon.0 v2:172.21.15.163:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559064a990e0 con 0x559064a77400 2024-06-18T03:00:37.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:37.699+0000 7fb4d9045040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T03:00:37.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:37 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/546001753' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T03:00:37.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:37 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T03:00:37.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:37 smithi163 ceph-mon[29791]: osdmap e104: 8 total, 8 up, 8 in 2024-06-18T03:00:37.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:37 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:03:00:37] ENGINE Bus STOPPING 2024-06-18T03:00:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:37 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/546001753' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T03:00:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:37 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-18T03:00:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:37 smithi186 ceph-mon[25944]: osdmap e104: 8 total, 8 up, 8 in 2024-06-18T03:00:38.149 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:37.855+0000 7fb4d9045040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T03:00:38.150 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:37 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:37.925+0000 7fb4d9045040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T03:00:38.173 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'sleep 180' 2024-06-18T03:00:38.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:03:00:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-18T03:00:38.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:03:00:38] ENGINE Bus STOPPED 2024-06-18T03:00:38.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:03:00:38] ENGINE Bus STARTING 2024-06-18T03:00:38.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:03:00:38] ENGINE Serving on http://:::9283 2024-06-18T03:00:38.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:03:00:38] ENGINE Bus STARTED 2024-06-18T03:00:38.490 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:38.182+0000 7fb4d9045040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T03:00:38.490 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:38.250+0000 7fb4d9045040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T03:00:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-18T03:00:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: mgrmap e31: x(active, starting, since 0.707829s) 2024-06-18T03:00:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:00:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:00:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:00:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T03:00:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T03:00:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: Manager daemon x is now available 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-18T03:00:38.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-18T03:00:38.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-18T03:00:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-18T03:00:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: mgrmap e31: x(active, starting, since 0.707829s) 2024-06-18T03:00:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:00:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:00:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:00:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T03:00:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T03:00:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T03:00:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T03:00:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T03:00:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T03:00:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T03:00:38.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T03:00:38.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T03:00:38.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T03:00:38.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T03:00:38.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T03:00:38.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: Manager daemon x is now available 2024-06-18T03:00:38.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:38.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-18T03:00:38.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-18T03:00:38.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-18T03:00:38.796 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:38 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-18T03:00:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-18T03:00:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: mgrmap e31: x(active, starting, since 0.707829s) 2024-06-18T03:00:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T03:00:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T03:00:38.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: Manager daemon x is now available 2024-06-18T03:00:38.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:38.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-18T03:00:38.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-18T03:00:38.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-18T03:00:38.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:38 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-18T03:00:39.434 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:39 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:39.252+0000 7fb4d9045040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T03:00:39.434 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:39 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:39.374+0000 7fb4d9045040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T03:00:39.701 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-mon[25944]: mgrmap e32: x(active, since 1.71456s) 2024-06-18T03:00:39.702 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-mon[25944]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:00:39.702 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:00:39.404+0000 7f8750288640 -1 log_channel(cephadm) log [ERR] : [18/Jun/2024:03:00:39] ENGINE Error in HTTPServer.serve 2024-06-18T03:00:39.702 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:00:39.702 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T03:00:39.702 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self._connections.run(self.expiration_interval) 2024-06-18T03:00:39.702 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T03:00:39.702 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self._run(expiration_interval) 2024-06-18T03:00:39.702 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T03:00:39.702 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T03:00:39.703 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T03:00:39.703 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T03:00:39.703 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T03:00:39.703 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: s = self.context.wrap_socket( 2024-06-18T03:00:39.703 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T03:00:39.703 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.sslsocket_class._create( 2024-06-18T03:00:39.703 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T03:00:39.703 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self.do_handshake() 2024-06-18T03:00:39.703 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T03:00:39.703 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self._sslobj.do_handshake() 2024-06-18T03:00:39.703 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T03:00:39.703 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:39 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:00:39.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:39 smithi163 ceph-mon[29791]: mgrmap e32: x(active, since 1.71456s) 2024-06-18T03:00:39.722 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:39 smithi163 ceph-mon[29791]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:00:39.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:39 smithi163 ceph-mon[25094]: mgrmap e32: x(active, since 1.71456s) 2024-06-18T03:00:39.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:39 smithi163 ceph-mon[25094]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:00:39.723 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:39 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:39.433+0000 7fb4d9045040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T03:00:39.723 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:39 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:39.665+0000 7fb4d9045040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T03:00:39.975 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:39 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:39.732+0000 7fb4d9045040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T03:00:39.976 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:39 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:39.810+0000 7fb4d9045040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T03:00:39.976 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:39 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:39.975+0000 7fb4d9045040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T03:00:40.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:40.431+0000 7fb4d9045040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T03:00:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:40 smithi186 ceph-mon[25944]: [18/Jun/2024:03:00:39] ENGINE Bus STARTING 2024-06-18T03:00:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:40 smithi186 ceph-mon[25944]: [18/Jun/2024:03:00:39] ENGINE Serving on https://172.21.15.186:7150 2024-06-18T03:00:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:40 smithi186 ceph-mon[25944]: [18/Jun/2024:03:00:39] ENGINE Error in HTTPServer.serve 2024-06-18T03:00:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:00:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._connections.run(self.expiration_interval) 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._run(expiration_interval) 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: s = self.context.wrap_socket( 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.sslsocket_class._create( 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: self.do_handshake() 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T03:00:40.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._sslobj.do_handshake() 2024-06-18T03:00:40.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T03:00:40.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:40 smithi186 ceph-mon[25944]: [18/Jun/2024:03:00:39] ENGINE Serving on http://172.21.15.186:8765 2024-06-18T03:00:40.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:40 smithi186 ceph-mon[25944]: [18/Jun/2024:03:00:39] ENGINE Bus STARTED 2024-06-18T03:00:40.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:40 smithi186 ceph-mon[25944]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:00:40.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:40 smithi186 ceph-mon[25944]: mgrmap e33: x(active, since 2s) 2024-06-18T03:00:40.846 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:40.777+0000 7fb4d9045040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T03:00:40.846 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:40.846+0000 7fb4d9045040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T03:00:40.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[29791]: [18/Jun/2024:03:00:39] ENGINE Bus STARTING 2024-06-18T03:00:40.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[29791]: [18/Jun/2024:03:00:39] ENGINE Serving on https://172.21.15.186:7150 2024-06-18T03:00:40.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[29791]: [18/Jun/2024:03:00:39] ENGINE Error in HTTPServer.serve 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._connections.run(self.expiration_interval) 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._run(expiration_interval) 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: s = self.context.wrap_socket( 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.sslsocket_class._create( 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T03:00:40.847 INFO:journalctl@ceph.mon.c.smithi163.stdout: self.do_handshake() 2024-06-18T03:00:40.848 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T03:00:40.848 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._sslobj.do_handshake() 2024-06-18T03:00:40.848 INFO:journalctl@ceph.mon.c.smithi163.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T03:00:40.848 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[29791]: [18/Jun/2024:03:00:39] ENGINE Serving on http://172.21.15.186:8765 2024-06-18T03:00:40.848 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[29791]: [18/Jun/2024:03:00:39] ENGINE Bus STARTED 2024-06-18T03:00:40.848 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[29791]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:00:40.848 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[29791]: mgrmap e33: x(active, since 2s) 2024-06-18T03:00:40.848 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[25094]: [18/Jun/2024:03:00:39] ENGINE Bus STARTING 2024-06-18T03:00:40.848 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[25094]: [18/Jun/2024:03:00:39] ENGINE Serving on https://172.21.15.186:7150 2024-06-18T03:00:40.848 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[25094]: [18/Jun/2024:03:00:39] ENGINE Error in HTTPServer.serve 2024-06-18T03:00:40.848 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:00:40.848 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._connections.run(self.expiration_interval) 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._run(expiration_interval) 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: s = self.context.wrap_socket( 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.sslsocket_class._create( 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: self.do_handshake() 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T03:00:40.849 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._sslobj.do_handshake() 2024-06-18T03:00:40.850 INFO:journalctl@ceph.mon.a.smithi163.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T03:00:40.850 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[25094]: [18/Jun/2024:03:00:39] ENGINE Serving on http://172.21.15.186:8765 2024-06-18T03:00:40.850 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[25094]: [18/Jun/2024:03:00:39] ENGINE Bus STARTED 2024-06-18T03:00:40.850 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[25094]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:00:40.850 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:40 smithi163 ceph-mon[25094]: mgrmap e33: x(active, since 2s) 2024-06-18T03:00:41.110 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:41 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:41.004+0000 7fb4d9045040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-18T03:00:41.409 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:41 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:41.336+0000 7fb4d9045040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T03:00:41.409 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:41 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:41.409+0000 7fb4d9045040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T03:00:41.731 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:41 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:41.638+0000 7fb4d9045040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T03:00:42.006 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:41 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:41.794+0000 7fb4d9045040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T03:00:42.006 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:41 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:41.866+0000 7fb4d9045040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T03:00:42.006 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:41 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:41.953+0000 7fb4d9045040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T03:00:42.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:42.017+0000 7fb4d9045040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T03:00:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:42 smithi186 ceph-mon[25944]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:00:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:42 smithi186 ceph-mon[25944]: mgrmap e34: x(active, since 4s) 2024-06-18T03:00:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:42 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:42.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-mon[25094]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:00:42.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-mon[25094]: mgrmap e34: x(active, since 4s) 2024-06-18T03:00:42.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:42.835 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:42.604+0000 7fb4d9045040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T03:00:42.836 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:42.775+0000 7fb4d9045040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T03:00:42.836 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:00:42] ENGINE Bus STARTING 2024-06-18T03:00:42.836 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: CherryPy Checker: 2024-06-18T03:00:42.836 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: The Application mounted at '' has an empty config. 2024-06-18T03:00:42.836 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:00:42.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-mon[29791]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:00:42.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-mon[29791]: mgrmap e34: x(active, since 4s) 2024-06-18T03:00:42.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:43.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:00:42] ENGINE Serving on http://:::9283 2024-06-18T03:00:43.293 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:42 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:00:42] ENGINE Bus STARTED 2024-06-18T03:00:43.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[29791]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:43.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:43.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[29791]: Standby manager daemon y started 2024-06-18T03:00:43.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-18T03:00:43.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T03:00:43.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-18T03:00:43.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T03:00:43.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:43.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:43.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[25094]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:43.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:43.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[25094]: Standby manager daemon y started 2024-06-18T03:00:43.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-18T03:00:43.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T03:00:43.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-18T03:00:43.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T03:00:43.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:43.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:43.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:43 smithi186 ceph-mon[25944]: from='client.25177 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:00:43.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:43 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:43.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:43 smithi186 ceph-mon[25944]: Standby manager daemon y started 2024-06-18T03:00:43.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:43 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-18T03:00:43.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:43 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T03:00:43.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:43 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-18T03:00:43.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:43 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T03:00:43.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:43 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:43.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:43 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:44.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:00:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T03:00:45.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[25094]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:00:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[25094]: mgrmap e35: x(active, since 6s), standbys: y 2024-06-18T03:00:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T03:00:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:00:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:00:45.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[29791]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:00:45.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[29791]: mgrmap e35: x(active, since 6s), standbys: y 2024-06-18T03:00:45.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T03:00:45.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:00:45.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:44 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:00:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:44 smithi186 ceph-mon[25944]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:00:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:44 smithi186 ceph-mon[25944]: mgrmap e35: x(active, since 6s), standbys: y 2024-06-18T03:00:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:44 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T03:00:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:44 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:44 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:44 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:44 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:44 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:00:45.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:44 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:00:45.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[29791]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T03:00:45.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[29791]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T03:00:45.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:00:45.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:00:45.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:00:45.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[25094]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T03:00:45.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[25094]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T03:00:45.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:45.873 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:00:45.873 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:00:45.873 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:45 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:00:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:45 smithi186 ceph-mon[25944]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T03:00:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:45 smithi186 ceph-mon[25944]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T03:00:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:45 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:46.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:45 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:46.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:45 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:46.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:45 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:46.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:45 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:46.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:45 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:00:46.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:45 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:00:46.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:45 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:00:46.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[29791]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:00:46.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[29791]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:00:46.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[29791]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:00:46.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[29791]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:00:46.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[29791]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:00:46.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[29791]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:00:46.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[29791]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:00:46.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[29791]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:00:46.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[29791]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-18T03:00:46.987 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[25094]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:00:46.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[25094]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:00:46.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[25094]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:00:46.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[25094]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:00:46.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[25094]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:00:46.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[25094]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:00:46.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[25094]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:00:46.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[25094]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:00:46.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:46 smithi163 ceph-mon[25094]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-18T03:00:47.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:46 smithi186 ceph-mon[25944]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:00:47.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:46 smithi186 ceph-mon[25944]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:00:47.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:46 smithi186 ceph-mon[25944]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:00:47.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:46 smithi186 ceph-mon[25944]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:00:47.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:46 smithi186 ceph-mon[25944]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:00:47.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:46 smithi186 ceph-mon[25944]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:00:47.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:46 smithi186 ceph-mon[25944]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:00:47.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:46 smithi186 ceph-mon[25944]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:00:47.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:46 smithi186 ceph-mon[25944]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-18T03:00:48.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:47 smithi163 ceph-mon[25094]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-18T03:00:48.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:47 smithi163 ceph-mon[29791]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-18T03:00:48.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:47 smithi186 ceph-mon[25944]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-18T03:00:48.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:00:48.118+0000 7f8783d2c640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:00:48.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Creating ceph-iscsi config... 2024-06-18T03:00:48.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:00:48.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:00:48.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:00:48.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:48.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:48.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:00:48.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:00:48.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return _run_code(code, main_globals, None, 2024-06-18T03:00:48.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:00:48.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: exec(code, run_globals) 2024-06-18T03:00:48.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:00:48.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:00:48.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:00:48.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:00:48.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:00:48.570 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:00:48.571 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:00:48.571 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:48.571 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:48.571 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:00:48.571 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:00:48.571 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.wait_async( 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return future.result(timeout) 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.__get_result() 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: raise self._exception 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: out, err, code = await self._run_cephadm( 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:00:48.572 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: raise OrchestratorError( 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Creating ceph-iscsi config... 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return _run_code(code, main_globals, None, 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: exec(code, run_globals) 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:00:48.573 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:00:48.574 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:00:48.574 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:00:48.574 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:00:48.574 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:00:48.574 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:00:48.574 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:48.574 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:48.958 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-mon[25944]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:00:48.958 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T03:00:48.958 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:00:48.958 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:00:48.958 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:00:48.958 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:48.958 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:48.958 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:00:48.958 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:00:48.958 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:00:48.959 INFO:journalctl@ceph.mon.b.smithi186.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.wait_async( 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: return future.result(timeout) 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.__get_result() 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise self._exception 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:00:48.960 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise OrchestratorError( 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:00:48.961 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:00:48.962 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:00:48.962 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:00:48.962 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:00:48.962 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:00:48.962 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:48.962 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:48.962 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-mon[25944]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T03:00:48.962 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:48 smithi186 ceph-mon[25944]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T03:00:49.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:48 smithi163 ceph-mon[29791]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:00:49.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:00:49.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:00:49.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:00:49.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:00:49.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:49.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:49.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:00:49.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:00:49.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:00:49.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:00:49.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:00:49.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.wait_async( 2024-06-18T03:00:49.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:00:49.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:00:49.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:00:49.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: return future.result(timeout) 2024-06-18T03:00:49.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:00:49.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:48 smithi163 ceph-mon[25094]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:00:49.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:00:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:00:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:00:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:00:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:00:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:00:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:00:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:00:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:00:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:00:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:00:49.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.wait_async( 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:00:49.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:00:49.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: return future.result(timeout) 2024-06-18T03:00:49.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:00:49.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.__get_result() 2024-06-18T03:00:49.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:00:49.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise self._exception 2024-06-18T03:00:49.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:00:49.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:00:49.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:00:49.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise OrchestratorError( 2024-06-18T03:00:49.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:00:49.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:00:49.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:00:49.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:00:49.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:00:49.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:49.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:49.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:00:49.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:00:49.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:00:49.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:00:49.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:00:49.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:00:49.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:00:49.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:00:49.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:00:49.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:00:49.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:00:49.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:00:49.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:49.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:49.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:48 smithi163 ceph-mon[29791]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T03:00:49.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:48 smithi163 ceph-mon[29791]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T03:00:49.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.__get_result() 2024-06-18T03:00:49.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:00:49.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise self._exception 2024-06-18T03:00:49.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:00:49.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:00:49.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:00:49.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise OrchestratorError( 2024-06-18T03:00:49.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:00:49.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:00:49.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:00:49.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:00:49.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:00:49.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:49.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:49.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:00:49.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:00:49.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:00:49.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:00:49.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:00:49.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:00:49.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:00:49.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:00:49.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:00:49.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:00:49.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:00:49.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:00:49.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:00:49.053 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:00:49.053 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:48 smithi163 ceph-mon[25094]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T03:00:49.053 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:48 smithi163 ceph-mon[25094]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T03:00:49.914 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:49 smithi186 ceph-mon[25944]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T03:00:50.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:49 smithi163 ceph-mon[29791]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T03:00:50.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:49 smithi163 ceph-mon[25094]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T03:00:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 systemd[1]: Stopping Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:00:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 bash[57360]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-18T03:00:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T03:00:51.281Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-18T03:00:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T03:00:51.281Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-06-18T03:00:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T03:00:51.281Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-06-18T03:00:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T03:00:51.281Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-06-18T03:00:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T03:00:51.281Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-06-18T03:00:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T03:00:51.281Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-06-18T03:00:51.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T03:00:51.282Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-18T03:00:51.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T03:00:51.282Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-06-18T03:00:51.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T03:00:51.282Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-06-18T03:00:51.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T03:00:51.282Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-06-18T03:00:51.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T03:00:51.282Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-06-18T03:00:51.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[54423]: ts=2024-06-18T03:00:51.282Z caller=main.go:1066 level=info msg="See you next time!" 2024-06-18T03:00:51.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 podman[57372]: 2024-06-18 03:00:51.292173298 +0000 UTC m=+0.131264552 container died eba156538173038e749e5fb353fe2ef8ca9543671cbbad420d68246701c77c5b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:00:51.932 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:51 smithi186 ceph-mon[25944]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T03:00:51.932 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 podman[57372]: 2024-06-18 03:00:51.681283923 +0000 UTC m=+0.520375176 container cleanup eba156538173038e749e5fb353fe2ef8ca9543671cbbad420d68246701c77c5b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:00:51.932 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 bash[57372]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a 2024-06-18T03:00:51.932 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 bash[57399]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-18T03:00:52.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:51 smithi163 ceph-mon[29791]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T03:00:52.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:51 smithi163 ceph-mon[25094]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T03:00:52.263 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 podman[57387]: 2024-06-18 03:00:51.932399559 +0000 UTC m=+0.642106828 container remove eba156538173038e749e5fb353fe2ef8ca9543671cbbad420d68246701c77c5b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:00:52.263 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-06-18T03:00:52.263 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 systemd[1]: Stopped Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:00:52.263 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a.service: Consumed 1.424s CPU time. 2024-06-18T03:00:52.263 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:51 smithi186 systemd[1]: Starting Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:00:52.264 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 podman[57466]: 2024-06-18 03:00:52.066284931 +0000 UTC m=+0.018429532 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-06-18T03:00:52.264 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 podman[57466]: 2024-06-18 03:00:52.166803888 +0000 UTC m=+0.118948489 container create 158474d404977558284e1da95c0acdee3e3effdf7d81fa93f24dce302d94e128 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:00:52.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 podman[57466]: 2024-06-18 03:00:52.377177074 +0000 UTC m=+0.329321676 container init 158474d404977558284e1da95c0acdee3e3effdf7d81fa93f24dce302d94e128 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:00:52.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 podman[57466]: 2024-06-18 03:00:52.37983329 +0000 UTC m=+0.331977891 container start 158474d404977558284e1da95c0acdee3e3effdf7d81fa93f24dce302d94e128 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:00:52.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.403Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-06-18T03:00:52.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.403Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-06-18T03:00:52.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.403Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-06-18T03:00:52.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.403Z caller=main.go:518 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi186 (none))" 2024-06-18T03:00:52.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.403Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-18T03:00:52.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.403Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-18T03:00:52.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.405Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-06-18T03:00:52.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.406Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-06-18T03:00:52.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.407Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-06-18T03:00:52.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.408Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-18T03:00:52.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.408Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.332µs 2024-06-18T03:00:52.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.408Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-18T03:00:52.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 bash[57466]: 158474d404977558284e1da95c0acdee3e3effdf7d81fa93f24dce302d94e128 2024-06-18T03:00:52.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 systemd[1]: Started Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:00:53.068 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.631Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-06-18T03:00:53.068 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:52.859Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-06-18T03:00:53.381 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:53.082Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-06-18T03:00:53.381 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:53.139Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-06-18T03:00:53.381 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:53.139Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-06-18T03:00:53.381 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:53.139Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.976µs wal_replay_duration=730.937389ms total_replay_duration=730.982057ms 2024-06-18T03:00:53.381 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:53.144Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-06-18T03:00:53.381 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:53.144Z caller=main.go:947 level=info msg="TSDB started" 2024-06-18T03:00:53.381 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:53.144Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-18T03:00:53.381 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:53.165Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.770521ms db_storage=765ns remote_storage=1.585µs web_handler=385ns query_engine=624ns scrape=369.773µs scrape_sd=123.248µs notify=14.587µs notify_sd=16.215µs rules=20.83854ms 2024-06-18T03:00:53.382 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:00:53.165Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-06-18T03:00:53.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:53 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:53.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:53 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:53.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:53 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:00:53.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:53 smithi163 ceph-mon[29791]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:00:53.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:53 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:53.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:53 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:53.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:53 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:53.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:53 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:00:53.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:53 smithi163 ceph-mon[25094]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:00:53.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:53 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:53.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:53.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:53.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:00:53.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-mon[25944]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:00:53.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:53 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:00:54.788 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:54 smithi186 ceph-mon[25944]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T03:00:54.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:54 smithi163 ceph-mon[29791]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T03:00:54.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:54 smithi163 ceph-mon[25094]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T03:00:56.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:56 smithi163 ceph-mon[29791]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T03:00:56.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:56 smithi163 ceph-mon[25094]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T03:00:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:56 smithi186 ceph-mon[25944]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T03:00:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:57 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:57 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:57 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:00:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:57 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:00:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:57 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:57 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:00:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:57 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:00:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:57 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:00:58.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:58.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:58.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:00:58.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:00:58.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:58.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:00:58.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:00:58.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:00:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:00:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:00:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:00:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:00:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:00:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:57 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:00:58.612 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:58 smithi163 ceph-mon[29791]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:00:58.613 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:58 smithi163 ceph-mon[29791]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:00:58.613 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:00:58 smithi163 ceph-mon[29791]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-18T03:00:58.613 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:58 smithi163 ceph-mon[25094]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:00:58.613 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:58 smithi163 ceph-mon[25094]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:00:58.613 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:00:58 smithi163 ceph-mon[25094]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-18T03:00:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:58 smithi186 ceph-mon[25944]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:00:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:58 smithi186 ceph-mon[25944]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:00:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:00:58 smithi186 ceph-mon[25944]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-18T03:01:00.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:00:59.914+0000 7f8783d2c640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:00.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:00.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:00.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:00.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:01:00.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Creating ceph-iscsi config... 2024-06-18T03:01:00.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:01:00.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:01:00.319 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:00.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:00.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:00.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:01:00.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:01:00.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return _run_code(code, main_globals, None, 2024-06-18T03:01:00.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:01:00.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: exec(code, run_globals) 2024-06-18T03:01:00.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:01:00.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:01:00.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:01:00.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:01:00.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:01:00.320 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:01:00.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:01:00.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:00.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:00.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:01:00.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:01:00.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:01:00.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:01:00.321 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.wait_async( 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return future.result(timeout) 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.__get_result() 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: raise self._exception 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: out, err, code = await self._run_cephadm( 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: raise OrchestratorError( 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:00.322 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:00.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:00.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:00.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:01:00.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Creating ceph-iscsi config... 2024-06-18T03:01:00.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:01:00.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:01:00.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:00.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:00.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:00.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:01:00.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:01:00.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return _run_code(code, main_globals, None, 2024-06-18T03:01:00.323 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:01:00.324 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: exec(code, run_globals) 2024-06-18T03:01:00.324 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:01:00.324 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:01:00.324 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:01:00.324 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:01:00.324 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:01:00.324 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:01:00.324 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:01:00.324 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:00.324 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:00:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:00 smithi186 ceph-mon[25944]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-18T03:01:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:00 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/2574454868' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T03:01:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:00 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3186007344' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3371340773"}]: dispatch 2024-06-18T03:01:01.038 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:00 smithi163 ceph-mon[29791]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-18T03:01:01.038 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:00 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2574454868' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T03:01:01.038 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:00 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3186007344' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3371340773"}]: dispatch 2024-06-18T03:01:01.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:00 smithi163 ceph-mon[25094]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-18T03:01:01.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:00 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2574454868' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T03:01:01.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:00 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3186007344' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3371340773"}]: dispatch 2024-06-18T03:01:01.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:01 smithi186 ceph-mon[25944]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:01.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:01.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:01.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:01.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:01:01.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T03:01:01.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:01:01.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:01:01.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:01.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:01.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:01.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:01:01.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:01:01.819 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:01:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.wait_async( 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: return future.result(timeout) 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.__get_result() 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise self._exception 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:01:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise OrchestratorError( 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:01:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:01 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3186007344' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3371340773"}]': finished 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:01 smithi186 ceph-mon[25944]: osdmap e105: 8 total, 8 up, 8 in 2024-06-18T03:01:01.823 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:01 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3955911171' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/3526437176"}]: dispatch 2024-06-18T03:01:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:01 smithi163 ceph-mon[29791]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:01:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:01:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:01:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:01:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:02.044 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.wait_async( 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: return future.result(timeout) 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.__get_result() 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise self._exception 2024-06-18T03:01:02.045 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise OrchestratorError( 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:02.046 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:02.047 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:01:02.047 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:01:02.047 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:01:02.047 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:01:02.047 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:01:02.047 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:01:02.047 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:01:02.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:01 smithi163 ceph-mon[25094]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:02.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:02.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:02.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:02.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:01:02.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:01:02.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:01:02.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:01:02.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:02.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:02.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:02.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:01:02.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:01:02.048 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:01:02.049 INFO:journalctl@ceph.mon.a.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.wait_async( 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: return future.result(timeout) 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.__get_result() 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise self._exception 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:01:02.050 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise OrchestratorError( 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi-foo-smithi163-ucaxbw 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: /usr/bin/podman: stderr Error: no such container ceph-929affee-2d1b-11ef-bc9d-c7b262605968-iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:01:02.051 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:01 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3186007344' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3371340773"}]': finished 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:01 smithi163 ceph-mon[25094]: osdmap e105: 8 total, 8 up, 8 in 2024-06-18T03:01:02.052 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:01 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3955911171' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/3526437176"}]: dispatch 2024-06-18T03:01:02.053 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:01:02.053 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:01:02.053 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:01:02.053 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:01:02.053 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:01:02.053 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:01:02.053 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:01:02.053 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:01 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3186007344' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3371340773"}]': finished 2024-06-18T03:01:02.053 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:01 smithi163 ceph-mon[29791]: osdmap e105: 8 total, 8 up, 8 in 2024-06-18T03:01:02.053 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:01 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3955911171' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/3526437176"}]: dispatch 2024-06-18T03:01:02.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:01:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:01:02] "GET /metrics HTTP/1.1" 200 37917 "" "Prometheus/2.33.4" 2024-06-18T03:01:03.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:02 smithi163 ceph-mon[29791]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:01:03.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:02 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3955911171' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/3526437176"}]': finished 2024-06-18T03:01:03.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:02 smithi163 ceph-mon[29791]: osdmap e106: 8 total, 8 up, 8 in 2024-06-18T03:01:03.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:02 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1916737288' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/4204946600"}]: dispatch 2024-06-18T03:01:03.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:02 smithi163 ceph-mon[25094]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:01:03.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:02 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3955911171' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/3526437176"}]': finished 2024-06-18T03:01:03.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:02 smithi163 ceph-mon[25094]: osdmap e106: 8 total, 8 up, 8 in 2024-06-18T03:01:03.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:02 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1916737288' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/4204946600"}]: dispatch 2024-06-18T03:01:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:02 smithi186 ceph-mon[25944]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:01:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:02 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3955911171' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/3526437176"}]': finished 2024-06-18T03:01:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:02 smithi186 ceph-mon[25944]: osdmap e106: 8 total, 8 up, 8 in 2024-06-18T03:01:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:02 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1916737288' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/4204946600"}]: dispatch 2024-06-18T03:01:04.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:03 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1916737288' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/4204946600"}]': finished 2024-06-18T03:01:04.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:03 smithi163 ceph-mon[29791]: osdmap e107: 8 total, 8 up, 8 in 2024-06-18T03:01:04.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:03 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/353756830' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/811039754"}]: dispatch 2024-06-18T03:01:04.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:03 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/811039754"}]: dispatch 2024-06-18T03:01:04.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:03 smithi163 ceph-mon[29791]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:01:04.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:03 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1916737288' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/4204946600"}]': finished 2024-06-18T03:01:04.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:03 smithi163 ceph-mon[25094]: osdmap e107: 8 total, 8 up, 8 in 2024-06-18T03:01:04.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:03 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/353756830' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/811039754"}]: dispatch 2024-06-18T03:01:04.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:03 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/811039754"}]: dispatch 2024-06-18T03:01:04.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:03 smithi163 ceph-mon[25094]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:01:04.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:03 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1916737288' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/4204946600"}]': finished 2024-06-18T03:01:04.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:03 smithi186 ceph-mon[25944]: osdmap e107: 8 total, 8 up, 8 in 2024-06-18T03:01:04.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:03 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/353756830' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/811039754"}]: dispatch 2024-06-18T03:01:04.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:03 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/811039754"}]: dispatch 2024-06-18T03:01:04.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:03 smithi186 ceph-mon[25944]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:01:05.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:04 smithi163 ceph-mon[29791]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/811039754"}]': finished 2024-06-18T03:01:05.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:04 smithi163 ceph-mon[29791]: osdmap e108: 8 total, 8 up, 8 in 2024-06-18T03:01:05.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:04 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3558166088' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2760671171"}]: dispatch 2024-06-18T03:01:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:04 smithi163 ceph-mon[25094]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/811039754"}]': finished 2024-06-18T03:01:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:04 smithi163 ceph-mon[25094]: osdmap e108: 8 total, 8 up, 8 in 2024-06-18T03:01:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:04 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3558166088' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2760671171"}]: dispatch 2024-06-18T03:01:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:04 smithi186 ceph-mon[25944]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/811039754"}]': finished 2024-06-18T03:01:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:04 smithi186 ceph-mon[25944]: osdmap e108: 8 total, 8 up, 8 in 2024-06-18T03:01:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:04 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3558166088' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2760671171"}]: dispatch 2024-06-18T03:01:06.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:05 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3558166088' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2760671171"}]': finished 2024-06-18T03:01:06.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:05 smithi163 ceph-mon[29791]: osdmap e109: 8 total, 8 up, 8 in 2024-06-18T03:01:06.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:05 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/4004997112' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3526437176"}]: dispatch 2024-06-18T03:01:06.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:05 smithi163 ceph-mon[29791]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:06.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:05 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3558166088' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2760671171"}]': finished 2024-06-18T03:01:06.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:05 smithi163 ceph-mon[25094]: osdmap e109: 8 total, 8 up, 8 in 2024-06-18T03:01:06.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:05 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/4004997112' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3526437176"}]: dispatch 2024-06-18T03:01:06.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:05 smithi163 ceph-mon[25094]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:06.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:05 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3558166088' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2760671171"}]': finished 2024-06-18T03:01:06.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:05 smithi186 ceph-mon[25944]: osdmap e109: 8 total, 8 up, 8 in 2024-06-18T03:01:06.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:05 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/4004997112' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3526437176"}]: dispatch 2024-06-18T03:01:06.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:05 smithi186 ceph-mon[25944]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:07.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:06 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/4004997112' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3526437176"}]': finished 2024-06-18T03:01:07.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:06 smithi163 ceph-mon[29791]: osdmap e110: 8 total, 8 up, 8 in 2024-06-18T03:01:07.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:06 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/4004997112' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3526437176"}]': finished 2024-06-18T03:01:07.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:06 smithi163 ceph-mon[25094]: osdmap e110: 8 total, 8 up, 8 in 2024-06-18T03:01:07.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:06 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/4004997112' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/3526437176"}]': finished 2024-06-18T03:01:07.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:06 smithi186 ceph-mon[25944]: osdmap e110: 8 total, 8 up, 8 in 2024-06-18T03:01:08.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:07 smithi163 ceph-mon[29791]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:01:08.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:07 smithi163 ceph-mon[25094]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:01:08.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:07 smithi186 ceph-mon[25944]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:01:09.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:08 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:01:09.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:08 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:01:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:08 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:01:09.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:09 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:09.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:09 smithi163 ceph-mon[25094]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-06-18T03:01:09.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:09 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:09.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:09 smithi163 ceph-mon[29791]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-06-18T03:01:10.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:09 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:10.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:09 smithi186 ceph-mon[25944]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-06-18T03:01:12.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:11 smithi163 ceph-mon[29791]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 643 B/s rd, 0 op/s 2024-06-18T03:01:12.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:11 smithi163 ceph-mon[25094]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 643 B/s rd, 0 op/s 2024-06-18T03:01:12.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:11 smithi186 ceph-mon[25944]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 643 B/s rd, 0 op/s 2024-06-18T03:01:12.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:01:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:01:12] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.33.4" 2024-06-18T03:01:14.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:13 smithi163 ceph-mon[29791]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:01:14.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:13 smithi163 ceph-mon[25094]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:01:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:13 smithi186 ceph-mon[25944]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:01:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:16 smithi163 ceph-mon[25094]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:01:16.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:16 smithi163 ceph-mon[29791]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:01:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:16 smithi186 ceph-mon[25944]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:01:18.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:17 smithi163 ceph-mon[25094]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 0 op/s 2024-06-18T03:01:18.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:17 smithi163 ceph-mon[29791]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 0 op/s 2024-06-18T03:01:18.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:17 smithi186 ceph-mon[25944]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 0 op/s 2024-06-18T03:01:19.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:19 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:19.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:19 smithi163 ceph-mon[29791]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:19 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:19 smithi163 ceph-mon[25094]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:20.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:19 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:20.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:19 smithi186 ceph-mon[25944]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:22.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:21 smithi163 ceph-mon[29791]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:22.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:21 smithi163 ceph-mon[25094]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:22.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:21 smithi186 ceph-mon[25944]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:22.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:01:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:01:22] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.33.4" 2024-06-18T03:01:23.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:23 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:01:23.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:23 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:01:23.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:23 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:01:24.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:24 smithi163 ceph-mon[29791]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:24.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:24 smithi163 ceph-mon[25094]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:24 smithi186 ceph-mon[25944]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:26.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:26 smithi163 ceph-mon[29791]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:26 smithi163 ceph-mon[25094]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:26 smithi186 ceph-mon[25944]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:27.952 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:27 smithi163 ceph-mon[29791]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:27.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:27 smithi163 ceph-mon[25094]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:28.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:27 smithi186 ceph-mon[25944]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:29 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:29 smithi163 ceph-mon[29791]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:29 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:29 smithi163 ceph-mon[25094]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:29 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:29 smithi186 ceph-mon[25944]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:32.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:31 smithi163 ceph-mon[29791]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:32.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:31 smithi163 ceph-mon[25094]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:32.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:31 smithi186 ceph-mon[25944]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:32.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:01:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:01:32] "GET /metrics HTTP/1.1" 200 37917 "" "Prometheus/2.33.4" 2024-06-18T03:01:34.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:33 smithi163 ceph-mon[29791]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:34.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:33 smithi163 ceph-mon[25094]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:33 smithi186 ceph-mon[25944]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:36 smithi163 ceph-mon[29791]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:36 smithi163 ceph-mon[25094]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:36 smithi186 ceph-mon[25944]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:37.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:37 smithi163 ceph-mon[29791]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:37 smithi163 ceph-mon[25094]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:38.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:37 smithi186 ceph-mon[25944]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:39.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:01:39.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:01:39.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:01:39.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:39 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:39.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:39 smithi163 ceph-mon[29791]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:39 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:39 smithi163 ceph-mon[25094]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:40.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:39 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:40.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:39 smithi186 ceph-mon[25944]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:42.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:41 smithi163 ceph-mon[29791]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:42.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:41 smithi163 ceph-mon[25094]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:42.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:41 smithi186 ceph-mon[25944]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:42.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:01:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:01:42] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-06-18T03:01:44.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:43 smithi163 ceph-mon[29791]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:44.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:43 smithi163 ceph-mon[25094]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:43 smithi186 ceph-mon[25944]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:46.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:46 smithi163 ceph-mon[29791]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:46.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:46 smithi163 ceph-mon[25094]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:46 smithi186 ceph-mon[25944]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:48.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:47 smithi163 ceph-mon[29791]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:48.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:47 smithi163 ceph-mon[25094]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:48.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:47 smithi186 ceph-mon[25944]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:49.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:49 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:49.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:49 smithi163 ceph-mon[29791]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:49.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:49 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:49.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:49 smithi163 ceph-mon[25094]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:50.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:49 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:50.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:49 smithi186 ceph-mon[25944]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:52.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:51 smithi163 ceph-mon[25094]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:52.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:51 smithi163 ceph-mon[29791]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:52.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:51 smithi186 ceph-mon[25944]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:52.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:01:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:01:52] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-06-18T03:01:53.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:53 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:01:53.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:53 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:01:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:53 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:01:54.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:54 smithi163 ceph-mon[29791]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:54.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:54 smithi163 ceph-mon[25094]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:54.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:54 smithi186 ceph-mon[25944]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:56.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:56 smithi163 ceph-mon[29791]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:56.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:56 smithi163 ceph-mon[25094]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:56 smithi186 ceph-mon[25944]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:58.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:57 smithi163 ceph-mon[29791]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:57 smithi163 ceph-mon[25094]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:58.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:57 smithi186 ceph-mon[25944]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:01:59.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:59 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:59.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:01:59 smithi163 ceph-mon[25094]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:01:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:59 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:01:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:01:59 smithi163 ceph-mon[29791]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:00.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:59 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:00.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:01:59 smithi186 ceph-mon[25944]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:01.020 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:02:01.020 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:02:01.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:02:01.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[29791]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:02:01.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:02:01.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:02:01.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:02:01.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[29791]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:02:01.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:02:01.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:02:01.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:02:01.022 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[25094]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:02:01.022 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:02:01.022 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:02:01.022 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:02:01.022 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:00 smithi163 ceph-mon[25094]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:02:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:00 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:02:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:00 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:02:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:00 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:02:01.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:00 smithi186 ceph-mon[25944]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:02:01.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:00 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:02:01.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:00 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:02:01.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:00 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:02:01.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:00 smithi186 ceph-mon[25944]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:02:01.900 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:01 smithi163 ceph-mon[29791]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:01.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:01 smithi163 ceph-mon[25094]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:01.931 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-mon[25944]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:02.294 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:02:01.931+0000 7f8783d2c640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:02:02.294 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Creating ceph-iscsi config... 2024-06-18T03:02:02.294 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:02:02.294 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return _run_code(code, main_globals, None, 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: exec(code, run_globals) 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:02:02.295 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:02:02.296 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:02:02.296 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:02.296 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:02.296 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:02:02.296 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:02:02.296 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:02:02.296 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:02:02.296 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.wait_async( 2024-06-18T03:02:02.296 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return future.result(timeout) 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.__get_result() 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: raise self._exception 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: out, err, code = await self._run_cephadm( 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: raise OrchestratorError( 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Creating ceph-iscsi config... 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:02:02.297 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return _run_code(code, main_globals, None, 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: exec(code, run_globals) 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:02:02.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:02:02.299 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:02:02.299 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:02.299 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:02.299 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:02:02] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-06-18T03:02:03.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:02 smithi163 ceph-mon[25094]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:02:03.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:02:03.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:02:03.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:02:03.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:02:03.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:03.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:03.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:02:03.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:02:03.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:02:03.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:02:03.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:02:03.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:02:03.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:02:03.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:02:03.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:02:03.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:02:03.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:02:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:02:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:02:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:02:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:02:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:02:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.wait_async( 2024-06-18T03:02:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:02:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:02:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:02:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: return future.result(timeout) 2024-06-18T03:02:03.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.__get_result() 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise self._exception 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise OrchestratorError( 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:03.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:03.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:03.048 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:02 smithi163 ceph-mon[29791]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:02:03.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:02:03.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:02:03.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:02:03.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:02:03.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:03.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:03.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:02:03.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:02:03.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:02:03.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:02:03.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:02:03.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.wait_async( 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: return future.result(timeout) 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.__get_result() 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise self._exception 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise OrchestratorError( 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:02:03.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:02:03.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:02:03.052 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:02:03.052 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:02:03.052 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:03.052 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:02 smithi186 ceph-mon[25944]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:02:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T03:02:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:02:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:02:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:02:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:02:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.wait_async( 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: return future.result(timeout) 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.__get_result() 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:02:03.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise self._exception 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise OrchestratorError( 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:02:03.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:02:03.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:02:03.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T03:02:03.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:02:03.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:02:03.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:02:03.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:02:03.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:02:03.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:02:03.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:02:03.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:02:03.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:02:04.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:03 smithi163 ceph-mon[29791]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:04.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:03 smithi163 ceph-mon[25094]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:04.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:03 smithi186 ceph-mon[25944]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:06.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:06 smithi163 ceph-mon[29791]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:06.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:06 smithi163 ceph-mon[25094]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:06 smithi186 ceph-mon[25944]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:07.924 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:07 smithi186 ceph-mon[25944]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:08.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:07 smithi163 ceph-mon[29791]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:08.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:07 smithi163 ceph-mon[25094]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:08 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:02:08.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:08 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:02:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:08 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:02:09.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:09 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:09.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:09 smithi163 ceph-mon[29791]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:09.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:09 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:09.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:09 smithi163 ceph-mon[25094]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:10.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:09 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:10.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:09 smithi186 ceph-mon[25944]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:12.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:11 smithi163 ceph-mon[29791]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:12.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:11 smithi163 ceph-mon[25094]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:12.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:11 smithi186 ceph-mon[25944]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:12.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:02:12] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-06-18T03:02:14.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:13 smithi163 ceph-mon[29791]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:14.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:13 smithi163 ceph-mon[25094]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:13 smithi186 ceph-mon[25944]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:16.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:16 smithi163 ceph-mon[29791]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:16 smithi163 ceph-mon[25094]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:16 smithi186 ceph-mon[25944]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:17 smithi186 ceph-mon[25944]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:18.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:17 smithi163 ceph-mon[29791]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:18.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:17 smithi163 ceph-mon[25094]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:19.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:19 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:19.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:19 smithi163 ceph-mon[29791]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:19.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:19 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:19.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:19 smithi163 ceph-mon[25094]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:20.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:19 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:20.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:19 smithi186 ceph-mon[25944]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:22.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:21 smithi163 ceph-mon[29791]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:22.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:21 smithi163 ceph-mon[25094]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:22.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:21 smithi186 ceph-mon[25944]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:22.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:02:22] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-06-18T03:02:23.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:23 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:02:23.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:23 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:02:23.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:23 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:02:24.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:24 smithi163 ceph-mon[29791]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:24.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:24 smithi163 ceph-mon[25094]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:24 smithi186 ceph-mon[25944]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:26 smithi186 ceph-mon[25944]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:02:27.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:26 smithi163 ceph-mon[25094]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:02:27.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:26 smithi163 ceph-mon[29791]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:02:28.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:27 smithi163 ceph-mon[29791]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:02:28.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:27 smithi163 ceph-mon[25094]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:02:28.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:27 smithi186 ceph-mon[25944]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:02:29.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:29 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:29.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:29 smithi163 ceph-mon[29791]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:29 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:29 smithi163 ceph-mon[25094]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:29 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:29 smithi186 ceph-mon[25944]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:32.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:31 smithi163 ceph-mon[25094]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:02:32.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:31 smithi163 ceph-mon[29791]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:02:32.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:31 smithi186 ceph-mon[25944]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:02:32.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:02:32] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.33.4" 2024-06-18T03:02:34.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:33 smithi163 ceph-mon[29791]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:34.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:33 smithi163 ceph-mon[25094]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:33 smithi186 ceph-mon[25944]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:36 smithi186 ceph-mon[25944]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:37.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:36 smithi163 ceph-mon[29791]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:37.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:36 smithi163 ceph-mon[25094]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:38.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:37 smithi163 ceph-mon[25094]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:38.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:37 smithi163 ceph-mon[29791]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:38.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:37 smithi186 ceph-mon[25944]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:39.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:02:39.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:02:39.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:02:39.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:39 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:39.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:39 smithi163 ceph-mon[29791]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:39 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:39 smithi163 ceph-mon[25094]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:40.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:39 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:40.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:39 smithi186 ceph-mon[25944]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:42.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:41 smithi163 ceph-mon[29791]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:42.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:41 smithi163 ceph-mon[25094]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:42.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:41 smithi186 ceph-mon[25944]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:42.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:02:42] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.33.4" 2024-06-18T03:02:44.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:43 smithi163 ceph-mon[29791]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:44.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:43 smithi163 ceph-mon[25094]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:43 smithi186 ceph-mon[25944]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:46 smithi186 ceph-mon[25944]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:47.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:46 smithi163 ceph-mon[29791]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:47.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:46 smithi163 ceph-mon[25094]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:48.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:47 smithi163 ceph-mon[29791]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:48.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:47 smithi163 ceph-mon[25094]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:48.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:47 smithi186 ceph-mon[25944]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:49.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:49 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:49.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:49 smithi163 ceph-mon[29791]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:49.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:49 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:49.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:49 smithi163 ceph-mon[25094]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:50.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:49 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:50.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:49 smithi186 ceph-mon[25944]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:52.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:51 smithi163 ceph-mon[29791]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:52.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:51 smithi163 ceph-mon[25094]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:52.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:51 smithi186 ceph-mon[25944]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:52.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:02:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:02:52] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.33.4" 2024-06-18T03:02:53.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:53 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:02:53.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:53 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:02:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:53 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:02:54.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:54 smithi163 ceph-mon[25094]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:54.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:54 smithi163 ceph-mon[29791]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:54.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:54 smithi186 ceph-mon[25944]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:56 smithi186 ceph-mon[25944]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:57.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:56 smithi163 ceph-mon[25094]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:57.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:56 smithi163 ceph-mon[29791]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:58.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:57 smithi163 ceph-mon[25094]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:58.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:57 smithi163 ceph-mon[29791]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:58.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:57 smithi186 ceph-mon[25944]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:02:59.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:59 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:59.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:02:59 smithi163 ceph-mon[29791]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:02:59.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:59 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:02:59.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:02:59 smithi163 ceph-mon[25094]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:00.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:59 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:00.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:02:59 smithi186 ceph-mon[25944]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:02.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:01 smithi163 ceph-mon[29791]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:02.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:01 smithi163 ceph-mon[25094]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:02.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:01 smithi186 ceph-mon[25944]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:02.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:03:02] "GET /metrics HTTP/1.1" 200 37927 "" "Prometheus/2.33.4" 2024-06-18T03:03:02.924 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:03:02.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:03:02.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:03:02.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[29791]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:03:02.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:03:02.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:03:02.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:03:02.925 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[29791]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:03:02.925 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:03:02.925 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:03:02.925 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:03:02.925 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[25094]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:03:02.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:03:02.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:03:02.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:03:02.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:02 smithi163 ceph-mon[25094]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:03:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:02 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:03:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:02 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:03:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:02 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:03:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:02 smithi186 ceph-mon[25944]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:03:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:02 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:03:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:02 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:03:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:02 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:03:03.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:02 smithi186 ceph-mon[25944]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:03:03.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:03 smithi163 ceph-mon[29791]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:03.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:03 smithi163 ceph-mon[25094]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:04.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-mon[25944]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:04.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:03.888+0000 7f8783d2c640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:04.069 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Creating ceph-iscsi config... 2024-06-18T03:03:04.069 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:04.069 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:04.069 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:04.069 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:04.069 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:04.069 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:03:04.069 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:04.069 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return _run_code(code, main_globals, None, 2024-06-18T03:03:04.069 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:04.069 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: exec(code, run_globals) 2024-06-18T03:03:04.070 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:04.070 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:04.070 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:04.070 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:04.070 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:04.070 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:04.070 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:04.070 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:04.070 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:04.071 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:03:04.071 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:03:04.071 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:03:04.071 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:03:04.071 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.wait_async( 2024-06-18T03:03:04.071 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:03:04.071 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:03:04.071 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:03:04.071 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return future.result(timeout) 2024-06-18T03:03:04.071 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:03:04.071 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.__get_result() 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: raise self._exception 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: out, err, code = await self._run_cephadm( 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: raise OrchestratorError( 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Creating ceph-iscsi config... 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:03:04.072 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:04.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return _run_code(code, main_globals, None, 2024-06-18T03:03:04.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:04.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: exec(code, run_globals) 2024-06-18T03:03:04.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:04.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:04.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:04.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:04.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:04.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:04.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:04.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:04.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:04 smithi163 ceph-mon[25094]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:03:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:03:05.043 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:03:05.044 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:03:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.wait_async( 2024-06-18T03:03:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:03:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:03:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:03:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: return future.result(timeout) 2024-06-18T03:03:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:03:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.__get_result() 2024-06-18T03:03:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:03:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise self._exception 2024-06-18T03:03:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:03:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:03:05.045 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:03:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise OrchestratorError( 2024-06-18T03:03:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:03:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:03:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:05.046 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:03:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:05.047 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:05.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:04 smithi163 ceph-mon[29791]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:05.047 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:03:05.047 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:05.048 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.wait_async( 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: return future.result(timeout) 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.__get_result() 2024-06-18T03:03:05.049 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise self._exception 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise OrchestratorError( 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:05.050 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:05.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:03:05.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:05.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:03:05.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:05.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:05.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:05.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:05.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:05.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:05.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:05.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:05.051 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:04 smithi186 ceph-mon[25944]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T03:03:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:03:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:03:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:03:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:03:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.wait_async( 2024-06-18T03:03:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:03:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:03:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:03:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: return future.result(timeout) 2024-06-18T03:03:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:03:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.__get_result() 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise self._exception 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise OrchestratorError( 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:03:05.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:05.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:03:05.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:05.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T03:03:05.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:05.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:05.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:05.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:05.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:05.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:05.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:05.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:05.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:06 smithi186 ceph-mon[25944]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:07.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:06 smithi163 ceph-mon[29791]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:07.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:06 smithi163 ceph-mon[25094]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:08.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:07 smithi163 ceph-mon[29791]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:08.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:07 smithi163 ceph-mon[25094]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:08.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:07 smithi186 ceph-mon[25944]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:09.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:08 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:09.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:08 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:08 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:09.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:09 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:09.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:09 smithi163 ceph-mon[29791]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:09.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:09 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:09.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:09 smithi163 ceph-mon[25094]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:10.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:09 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:10.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:09 smithi186 ceph-mon[25944]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:12.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:11 smithi163 ceph-mon[29791]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:12.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:11 smithi163 ceph-mon[25094]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:12.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:11 smithi186 ceph-mon[25944]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:12.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:03:12] "GET /metrics HTTP/1.1" 200 37925 "" "Prometheus/2.33.4" 2024-06-18T03:03:14.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:13 smithi163 ceph-mon[29791]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:14.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:13 smithi163 ceph-mon[25094]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:14.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:13 smithi186 ceph-mon[25944]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:16 smithi186 ceph-mon[25944]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:17.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:16 smithi163 ceph-mon[29791]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:17.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:16 smithi163 ceph-mon[25094]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:18.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:17 smithi163 ceph-mon[29791]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:18.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:17 smithi163 ceph-mon[25094]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:18.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:17 smithi186 ceph-mon[25944]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:19.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:19 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:19.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:19 smithi163 ceph-mon[29791]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:19 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:19 smithi163 ceph-mon[25094]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:20.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:19 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:20.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:19 smithi186 ceph-mon[25944]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:22.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:21 smithi163 ceph-mon[29791]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:22.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:21 smithi163 ceph-mon[25094]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:22.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:21 smithi186 ceph-mon[25944]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:22.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:22 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:03:22] "GET /metrics HTTP/1.1" 200 37925 "" "Prometheus/2.33.4" 2024-06-18T03:03:23.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:23 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:23.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:23 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:23.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:23 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:24.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:24 smithi163 ceph-mon[29791]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:24.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:24 smithi163 ceph-mon[25094]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:24 smithi186 ceph-mon[25944]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:26 smithi186 ceph-mon[25944]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:27.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:26 smithi163 ceph-mon[29791]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:27.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:26 smithi163 ceph-mon[25094]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:28.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:27 smithi163 ceph-mon[25094]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:28.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:27 smithi163 ceph-mon[29791]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:28.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:27 smithi186 ceph-mon[25944]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:29 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:29 smithi163 ceph-mon[29791]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:29 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:29 smithi163 ceph-mon[25094]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:29 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:29 smithi186 ceph-mon[25944]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:32.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:31 smithi163 ceph-mon[29791]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:32.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:31 smithi163 ceph-mon[25094]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:32.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:31 smithi186 ceph-mon[25944]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:32.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:32 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:03:32] "GET /metrics HTTP/1.1" 200 37919 "" "Prometheus/2.33.4" 2024-06-18T03:03:34.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:33 smithi163 ceph-mon[29791]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:34.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:33 smithi163 ceph-mon[25094]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:33 smithi186 ceph-mon[25944]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:36 smithi186 ceph-mon[25944]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:37.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:36 smithi163 ceph-mon[29791]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:37.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:36 smithi163 ceph-mon[25094]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:38.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:37 smithi163 ceph-mon[29791]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:38.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:37 smithi163 ceph-mon[25094]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:38.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:37 smithi186 ceph-mon[25944]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:39.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:38 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:39.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:38 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:39.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:38 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:39.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:39 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:39.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:39 smithi163 ceph-mon[29791]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:39 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:39 smithi163 ceph-mon[25094]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:40.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:39 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:40.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:39 smithi186 ceph-mon[25944]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:40.402 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps' 2024-06-18T03:03:41.946 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:41 smithi163 ceph-mon[29791]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:41.946 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:41 smithi163 ceph-mon[25094]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:42.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:41 smithi186 ceph-mon[25944]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:42.403 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:03:42.403 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (12m) 2m ago 20m 22.1M - 0.23.0 ba2b418f427c 5e109a483ef4 2024-06-18T03:03:42.403 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (12m) 2m ago 20m 52.9M - 8.3.5 dad864ee21e9 a3fd9ade8374 2024-06-18T03:03:42.404 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 running (6m) 2m ago 19m 53.1M - 3.5 e1d6a67b021e 2b1dde02e840 2024-06-18T03:03:42.404 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283 running (15m) 2m ago 24m 488M - 19.0.0-3124-g3781e3a5 af93c036e016 d7bd16a633ff 2024-06-18T03:03:42.404 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (9m) 2m ago 26m 213M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:03:42.404 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (26m) 2m ago 26m 74.7M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T03:03:42.404 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (25m) 2m ago 25m 51.8M 2048M 17.2.0 e1d6a67b021e a3618f734a9f 2024-06-18T03:03:42.404 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (25m) 2m ago 25m 50.7M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T03:03:42.404 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (12m) 2m ago 21m 16.0M - 1.3.1 1dbe0e931976 8d0aecfcef8a 2024-06-18T03:03:42.404 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (12m) 2m ago 21m 17.3M - 1.3.1 1dbe0e931976 0592fede9474 2024-06-18T03:03:42.404 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (24m) 2m ago 24m 54.7M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T03:03:42.404 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (24m) 2m ago 24m 54.6M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T03:03:42.404 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (23m) 2m ago 23m 50.4M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:03:42.404 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (23m) 2m ago 23m 53.1M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:03:42.404 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (23m) 2m ago 23m 58.9M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:03:42.405 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (22m) 2m ago 22m 51.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:03:42.405 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (22m) 2m ago 22m 50.1M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:03:42.405 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (21m) 2m ago 21m 53.8M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:03:42.405 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (2m) 2m ago 21m 35.5M - 2.33.4 514e6a882f6e 158474d40497 2024-06-18T03:03:42.405 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (19m) 2m ago 19m 77.6M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:03:42.405 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (19m) 2m ago 19m 78.5M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:03:42.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:03:42] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.33.4" 2024-06-18T03:03:43.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:42 smithi163 ceph-mon[25094]: from='client.25315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:43.040 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:42 smithi163 ceph-mon[29791]: from='client.25315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:43.063 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ls' 2024-06-18T03:03:43.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:42 smithi186 ceph-mon[25944]: from='client.25315 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:44.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:43 smithi163 ceph-mon[29791]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:44.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:43 smithi163 ceph-mon[25094]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:43 smithi186 ceph-mon[25944]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:45.042 INFO:teuthology.orchestra.run.smithi163.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-06-18T03:03:45.043 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager ?:9093,9094 1/1 3m ago 21m smithi163=a;count:1 2024-06-18T03:03:45.043 INFO:teuthology.orchestra.run.smithi163.stdout:grafana ?:3000 1/1 2m ago 21m smithi186=a;count:1 2024-06-18T03:03:45.043 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo ?:5000 1/1 3m ago 19m count:1 2024-06-18T03:03:45.043 INFO:teuthology.orchestra.run.smithi163.stdout:mgr 2/2 3m ago 24m smithi163=y;smithi186=x;count:2 2024-06-18T03:03:45.043 INFO:teuthology.orchestra.run.smithi163.stdout:mon 3/3 3m ago 25m smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi186:172.21.15.186=b;count:3 2024-06-18T03:03:45.043 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter ?:9100 2/2 3m ago 21m smithi163=a;smithi186=b;count:2 2024-06-18T03:03:45.043 INFO:teuthology.orchestra.run.smithi163.stdout:osd 8 3m ago - 2024-06-18T03:03:45.043 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus ?:9095 1/1 2m ago 21m smithi186=a;count:1 2024-06-18T03:03:45.043 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo ?:8000 2/2 3m ago 19m count:2 2024-06-18T03:03:45.662 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions' 2024-06-18T03:03:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:46 smithi186 ceph-mon[25944]: from='client.25321 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:46 smithi186 ceph-mon[25944]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:46.941 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:46 smithi163 ceph-mon[29791]: from='client.25321 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:46.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:46 smithi163 ceph-mon[29791]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:46.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:46 smithi163 ceph-mon[25094]: from='client.25321 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:46.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:46 smithi163 ceph-mon[25094]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:47.591 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:03:47.592 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:03:47.592 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-18T03:03:47.592 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:03:47.592 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:03:47.592 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:03:47.592 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:03:47.592 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:03:47.592 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-18T03:03:47.592 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:03:47.592 INFO:teuthology.orchestra.run.smithi163.stdout: "mds": {}, 2024-06-18T03:03:47.592 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:03:47.592 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:03:47.593 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:03:47.593 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:03:47.593 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-18T03:03:47.593 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:03:47.593 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:03:47.593 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:03:48.026 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:47 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3550381809' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:03:48.026 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:47 smithi163 ceph-mon[29791]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:48.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:47 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3550381809' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:03:48.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:47 smithi163 ceph-mon[25094]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:48.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:47 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3550381809' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:03:48.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:47 smithi186 ceph-mon[25944]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:48.220 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-06-18T03:03:49.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:49 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:49.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:49 smithi163 ceph-mon[29791]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:49.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:49 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:49.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:49 smithi163 ceph-mon[25094]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:50.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:49 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:03:50.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:49 smithi186 ceph-mon[25944]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:51.396 INFO:teuthology.orchestra.run.smithi163.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:03:51.767 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[25094]: from='client.15429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:51.767 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:03:51.768 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:03:51.768 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:03:51.768 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:03:51.768 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:03:51.768 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:03:51.768 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:03:51.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[29791]: from='client.15429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:51.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:03:51.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:03:51.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:03:51.769 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:03:51.769 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:03:51.769 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:03:51.769 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:51 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:03:51.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:51 smithi186 ceph-mon[25944]: from='client.15429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:51.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:51 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:03:51.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:51 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:03:51.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:51 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:03:51.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:51 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:03:51.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:51 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:03:51.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:51 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:03:51.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:51 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:03:52.058 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-18T03:03:52.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:03:52] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.33.4" 2024-06-18T03:03:52.960 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:52 smithi163 ceph-mon[25094]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:03:52.960 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:52 smithi163 ceph-mon[25094]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:03:52.960 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:52 smithi163 ceph-mon[25094]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:03:52.960 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:52 smithi163 ceph-mon[25094]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:52.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:52 smithi163 ceph-mon[29791]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:03:52.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:52 smithi163 ceph-mon[29791]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:03:52.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:52 smithi163 ceph-mon[29791]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:03:52.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:52 smithi163 ceph-mon[29791]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:52 smithi186 ceph-mon[25944]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:03:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:52 smithi186 ceph-mon[25944]: Reconfiguring iscsi.foo.smithi163.ucaxbw (dependencies changed)... 2024-06-18T03:03:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:52 smithi186 ceph-mon[25944]: Reconfiguring daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:03:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:52 smithi186 ceph-mon[25944]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:53.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:53.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:53.363+0000 7f8783d2c640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:53.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Creating ceph-iscsi config... 2024-06-18T03:03:53.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:53.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:53.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:53.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:53.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:53.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:03:53.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:53.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return _run_code(code, main_globals, None, 2024-06-18T03:03:53.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:53.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: exec(code, run_globals) 2024-06-18T03:03:53.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:53.820 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:53.820 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:53.820 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:53.820 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:53.820 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:53.820 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:53.820 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:53.820 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:53.821 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:03:53.821 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:03:53.821 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:03:53.821 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:03:53.821 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.wait_async( 2024-06-18T03:03:53.821 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:03:53.821 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:03:53.821 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:03:53.821 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return future.result(timeout) 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return self.__get_result() 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: raise self._exception 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: out, err, code = await self._run_cephadm( 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: raise OrchestratorError( 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Creating ceph-iscsi config... 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:53.822 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:53.823 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:53.823 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: Traceback (most recent call last): 2024-06-18T03:03:53.823 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:53.823 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: return _run_code(code, main_globals, None, 2024-06-18T03:03:53.823 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:53.823 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: exec(code, run_globals) 2024-06-18T03:03:53.823 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:53.823 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:53.823 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:53.823 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:53.824 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:53.824 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:53.824 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:53.824 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:53.824 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:53.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:53 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:53.846 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:53 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:54.718 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:54 smithi163 ceph-mon[25094]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:03:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:03:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:03:54.719 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.wait_async( 2024-06-18T03:03:54.720 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: return future.result(timeout) 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.__get_result() 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise self._exception 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: raise OrchestratorError( 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:54.721 INFO:journalctl@ceph.mon.a.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:54.722 INFO:journalctl@ceph.mon.a.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:54.722 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:54.722 INFO:journalctl@ceph.mon.a.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:54.722 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:54.722 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:54.722 INFO:journalctl@ceph.mon.a.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:03:54.722 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:54.722 INFO:journalctl@ceph.mon.a.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:03:54.722 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:54.722 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:54.722 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:54.722 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:54.722 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:54.723 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:54.723 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:54.723 INFO:journalctl@ceph.mon.a.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:54.723 INFO:journalctl@ceph.mon.a.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:54.723 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:54 smithi163 ceph-mon[25094]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:03:54.723 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:54 smithi163 ceph-mon[25094]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:54.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:54 smithi163 ceph-mon[29791]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:54.723 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:03:54.723 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:54.723 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:54.724 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.wait_async( 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: return future.result(timeout) 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:03:54.725 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.__get_result() 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise self._exception 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: raise OrchestratorError( 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: Creating ceph-iscsi config... 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:54.726 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:54 smithi163 ceph-mon[29791]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:03:54.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:54 smithi163 ceph-mon[29791]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:55.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:54 smithi186 ceph-mon[25944]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:55.069 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.wait_async( 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-18T03:03:55.070 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: return future.result(timeout) 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.__get_result() 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise self._exception 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: out, err, code = await self._run_cephadm( 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: raise OrchestratorError( 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Creating ceph-iscsi config... 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:03:55.071 INFO:journalctl@ceph.mon.b.smithi186.stdout: Non-zero exit code 1 from systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: exec(code, run_globals) 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-18T03:03:55.072 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:03:55.073 INFO:journalctl@ceph.mon.b.smithi186.stdout: RuntimeError: Failed command: systemctl restart ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:03:55.073 INFO:journalctl@ceph.mon.b.smithi186.stdout: See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:03:55.073 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:54 smithi186 ceph-mon[25944]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:03:55.073 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:54 smithi186 ceph-mon[25944]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:03:55.174 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:03:55.831 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:03:55.832 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (12m) 3m ago 20m 22.1M - 0.23.0 ba2b418f427c 5e109a483ef4 2024-06-18T03:03:55.832 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (12m) 2m ago 20m 52.9M - 8.3.5 dad864ee21e9 a3fd9ade8374 2024-06-18T03:03:55.832 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 running (6m) 3m ago 19m 53.1M - 3.5 e1d6a67b021e 2b1dde02e840 2024-06-18T03:03:55.832 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283 running (15m) 2m ago 25m 488M - 19.0.0-3124-g3781e3a5 af93c036e016 d7bd16a633ff 2024-06-18T03:03:55.832 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (9m) 3m ago 26m 213M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:03:55.832 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (26m) 3m ago 26m 74.7M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T03:03:55.832 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (25m) 2m ago 25m 51.8M 2048M 17.2.0 e1d6a67b021e a3618f734a9f 2024-06-18T03:03:55.832 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (25m) 3m ago 25m 50.7M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T03:03:55.832 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (12m) 3m ago 21m 16.0M - 1.3.1 1dbe0e931976 8d0aecfcef8a 2024-06-18T03:03:55.832 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (12m) 2m ago 21m 17.3M - 1.3.1 1dbe0e931976 0592fede9474 2024-06-18T03:03:55.832 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (24m) 3m ago 24m 54.7M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T03:03:55.832 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (24m) 3m ago 24m 54.6M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T03:03:55.832 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (24m) 3m ago 24m 50.4M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:03:55.833 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (23m) 3m ago 23m 53.1M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:03:55.834 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (23m) 2m ago 23m 58.9M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:03:55.834 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (22m) 2m ago 22m 51.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:03:55.834 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (22m) 2m ago 22m 50.1M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:03:55.834 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (22m) 2m ago 22m 53.8M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:03:55.835 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (3m) 2m ago 21m 35.5M - 2.33.4 514e6a882f6e 158474d40497 2024-06-18T03:03:55.835 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (19m) 3m ago 19m 77.6M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:03:55.835 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (19m) 2m ago 19m 78.5M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:03:56.226 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:03:56.226 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:03:56.226 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-18T03:03:56.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:03:56.226 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:03:56.226 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:03:56.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:03:56.226 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:03:56.227 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-18T03:03:56.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:03:56.227 INFO:teuthology.orchestra.run.smithi163.stdout: "mds": {}, 2024-06-18T03:03:56.227 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:03:56.227 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:03:56.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:03:56.227 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:03:56.227 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-18T03:03:56.227 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:03:56.227 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:03:56.227 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:03:56.554 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:03:56.555 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:03:56.555 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:03:56.555 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-06-18T03:03:56.555 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [ 2024-06-18T03:03:56.555 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr" 2024-06-18T03:03:56.555 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2024-06-18T03:03:56.555 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "2/2 daemons upgraded", 2024-06-18T03:03:56.555 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 image", 2024-06-18T03:03:56.555 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:03:56.555 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:03:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:56 smithi186 ceph-mon[25944]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:56 smithi186 ceph-mon[25944]: from='client.25345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:56 smithi186 ceph-mon[25944]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:56 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:03:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:56 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:03:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:56 smithi186 ceph-mon[25944]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-06-18T03:03:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:56 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-06-18T03:03:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:56 smithi186 ceph-mon[25944]: osdmap e111: 8 total, 8 up, 8 in 2024-06-18T03:03:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:56 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/638887259' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:03:56.991 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[29791]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:56.991 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[29791]: from='client.25345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:56.991 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[29791]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:56.991 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:03:56.991 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:03:56.991 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[29791]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-06-18T03:03:56.991 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-06-18T03:03:56.991 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[29791]: osdmap e111: 8 total, 8 up, 8 in 2024-06-18T03:03:56.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/638887259' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:03:56.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[25094]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:56.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[25094]: from='client.25345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:56.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[25094]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:03:56.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' 2024-06-18T03:03:56.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:03:56.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[25094]: from='mgr.24872 172.21.15.186:0/1061595099' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-06-18T03:03:56.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-06-18T03:03:56.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[25094]: osdmap e111: 8 total, 8 up, 8 in 2024-06-18T03:03:56.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/638887259' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:03:57.267 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:56.982+0000 7f87edf60640 -1 mgr handle_mgr_map I was active but no longer am 2024-06-18T03:03:57.267 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ignoring --setuser ceph since I am not root 2024-06-18T03:03:57.267 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ignoring --setgroup ceph since I am not root 2024-06-18T03:03:57.267 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mgr[45515]: -- 172.21.15.186:0/48882567 <== mon.2 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56455f2b70e0 con 0x56455f295000 2024-06-18T03:03:57.267 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:57.138+0000 7f9cc1cd6040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T03:03:57.279 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:03:56] ENGINE Bus STOPPING 2024-06-18T03:03:57.280 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:03:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-18T03:03:57.280 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:03:57] ENGINE Bus STOPPED 2024-06-18T03:03:57.280 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:03:57] ENGINE Bus STARTING 2024-06-18T03:03:57.280 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:03:57] ENGINE Serving on http://:::9283 2024-06-18T03:03:57.280 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:03:57] ENGINE Bus STARTED 2024-06-18T03:03:57.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:57.289+0000 7f9cc1cd6040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T03:03:57.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:57.359+0000 7f9cc1cd6040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T03:03:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='client.25351 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:03:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:03:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: Upgrade: Need to upgrade myself (mgr.x) 2024-06-18T03:03:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: Upgrade: Need to upgrade myself (mgr.x) 2024-06-18T03:03:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: Failing over to other MGR 2024-06-18T03:03:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='client.25363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: Standby manager daemon x started 2024-06-18T03:03:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.24872 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-06-18T03:03:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: mgrmap e36: y(active, starting, since 1.00293s), standbys: x 2024-06-18T03:03:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:03:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:03:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:03:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T03:03:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T03:03:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T03:03:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T03:03:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T03:03:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T03:03:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T03:03:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T03:03:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T03:03:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T03:03:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T03:03:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T03:03:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T03:03:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: Manager daemon y is now available 2024-06-18T03:03:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T03:03:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T03:03:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='client.25351 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:03:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:03:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: Upgrade: Need to upgrade myself (mgr.x) 2024-06-18T03:03:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: Upgrade: Need to upgrade myself (mgr.x) 2024-06-18T03:03:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: Failing over to other MGR 2024-06-18T03:03:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='client.25363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: Standby manager daemon x started 2024-06-18T03:03:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.24872 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-06-18T03:03:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: mgrmap e36: y(active, starting, since 1.00293s), standbys: x 2024-06-18T03:03:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:03:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:03:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:03:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T03:03:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T03:03:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T03:03:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T03:03:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T03:03:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T03:03:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T03:03:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T03:03:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T03:03:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T03:03:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T03:03:58.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T03:03:58.048 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T03:03:58.048 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: Manager daemon y is now available 2024-06-18T03:03:58.048 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:58.048 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T03:03:58.048 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T03:03:58.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:57.621+0000 7f9cc1cd6040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T03:03:58.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:57.686+0000 7f9cc1cd6040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T03:03:58.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='client.25351 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:58.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:03:58.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:03:58.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: Upgrade: Need to upgrade myself (mgr.x) 2024-06-18T03:03:58.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: Upgrade: Need to upgrade myself (mgr.x) 2024-06-18T03:03:58.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: Failing over to other MGR 2024-06-18T03:03:58.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='client.25363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:03:58.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: Standby manager daemon x started 2024-06-18T03:03:58.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.24872 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: mgrmap e36: y(active, starting, since 1.00293s), standbys: x 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T03:03:58.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T03:03:58.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T03:03:58.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T03:03:58.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T03:03:58.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: Manager daemon y is now available 2024-06-18T03:03:58.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:03:58.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T03:03:58.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T03:03:58.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:03:58.620+0000 7fb43630c640 -1 log_channel(cephadm) log [ERR] : [18/Jun/2024:03:03:58] ENGINE Error in HTTPServer.serve 2024-06-18T03:03:58.793 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Traceback (most recent call last): 2024-06-18T03:03:58.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T03:03:58.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: self._connections.run(self.expiration_interval) 2024-06-18T03:03:58.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T03:03:58.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: self._run(expiration_interval) 2024-06-18T03:03:58.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T03:03:58.794 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T03:03:58.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T03:03:58.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T03:03:58.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T03:03:58.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: s = self.context.wrap_socket( 2024-06-18T03:03:58.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T03:03:58.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: return self.sslsocket_class._create( 2024-06-18T03:03:58.795 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T03:03:58.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: self.do_handshake() 2024-06-18T03:03:58.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T03:03:58.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: self._sslobj.do_handshake() 2024-06-18T03:03:58.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T03:03:58.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:03:58.985 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:58.699+0000 7f9cc1cd6040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T03:03:58.985 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:58.821+0000 7f9cc1cd6040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T03:03:58.985 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:58.885+0000 7f9cc1cd6040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T03:03:59.254 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:59.114+0000 7f9cc1cd6040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T03:03:59.254 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:59.179+0000 7f9cc1cd6040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T03:03:59.255 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:59.254+0000 7f9cc1cd6040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T03:03:59.255 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:58 smithi186 ceph-mon[25944]: mgrmap e37: y(active, since 2s), standbys: x 2024-06-18T03:03:59.255 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:58 smithi186 ceph-mon[25944]: [18/Jun/2024:03:03:58] ENGINE Bus STARTING 2024-06-18T03:03:59.255 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:58 smithi186 ceph-mon[25944]: [18/Jun/2024:03:03:58] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T03:03:59.255 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:58 smithi186 ceph-mon[25944]: [18/Jun/2024:03:03:58] ENGINE Error in HTTPServer.serve 2024-06-18T03:03:59.255 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:03:59.255 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T03:03:59.255 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._connections.run(self.expiration_interval) 2024-06-18T03:03:59.255 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T03:03:59.255 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._run(expiration_interval) 2024-06-18T03:03:59.255 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T03:03:59.255 INFO:journalctl@ceph.mon.b.smithi186.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T03:03:59.256 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T03:03:59.256 INFO:journalctl@ceph.mon.b.smithi186.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T03:03:59.256 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T03:03:59.256 INFO:journalctl@ceph.mon.b.smithi186.stdout: s = self.context.wrap_socket( 2024-06-18T03:03:59.256 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T03:03:59.256 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.sslsocket_class._create( 2024-06-18T03:03:59.256 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T03:03:59.256 INFO:journalctl@ceph.mon.b.smithi186.stdout: self.do_handshake() 2024-06-18T03:03:59.256 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T03:03:59.256 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._sslobj.do_handshake() 2024-06-18T03:03:59.256 INFO:journalctl@ceph.mon.b.smithi186.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T03:03:59.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-mon[25094]: mgrmap e37: y(active, since 2s), standbys: x 2024-06-18T03:03:59.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-mon[25094]: [18/Jun/2024:03:03:58] ENGINE Bus STARTING 2024-06-18T03:03:59.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-mon[25094]: [18/Jun/2024:03:03:58] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T03:03:59.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-mon[25094]: [18/Jun/2024:03:03:58] ENGINE Error in HTTPServer.serve 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._connections.run(self.expiration_interval) 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._run(expiration_interval) 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: s = self.context.wrap_socket( 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.sslsocket_class._create( 2024-06-18T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T03:03:59.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: self.do_handshake() 2024-06-18T03:03:59.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T03:03:59.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._sslobj.do_handshake() 2024-06-18T03:03:59.294 INFO:journalctl@ceph.mon.a.smithi163.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T03:03:59.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-mon[29791]: mgrmap e37: y(active, since 2s), standbys: x 2024-06-18T03:03:59.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-mon[29791]: [18/Jun/2024:03:03:58] ENGINE Bus STARTING 2024-06-18T03:03:59.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-mon[29791]: [18/Jun/2024:03:03:58] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T03:03:59.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:58 smithi163 ceph-mon[29791]: [18/Jun/2024:03:03:58] ENGINE Error in HTTPServer.serve 2024-06-18T03:03:59.294 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:03:59.294 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T03:03:59.294 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._connections.run(self.expiration_interval) 2024-06-18T03:03:59.295 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T03:03:59.295 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._run(expiration_interval) 2024-06-18T03:03:59.295 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T03:03:59.295 INFO:journalctl@ceph.mon.c.smithi163.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T03:03:59.295 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T03:03:59.295 INFO:journalctl@ceph.mon.c.smithi163.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T03:03:59.295 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T03:03:59.295 INFO:journalctl@ceph.mon.c.smithi163.stdout: s = self.context.wrap_socket( 2024-06-18T03:03:59.295 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T03:03:59.295 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.sslsocket_class._create( 2024-06-18T03:03:59.295 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T03:03:59.295 INFO:journalctl@ceph.mon.c.smithi163.stdout: self.do_handshake() 2024-06-18T03:03:59.296 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T03:03:59.296 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._sslobj.do_handshake() 2024-06-18T03:03:59.296 INFO:journalctl@ceph.mon.c.smithi163.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T03:03:59.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:59.424+0000 7f9cc1cd6040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T03:03:59.985 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:03:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:03:59.879+0000 7f9cc1cd6040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T03:04:00.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:59 smithi163 ceph-mon[25094]: [18/Jun/2024:03:03:58] ENGINE Serving on http://172.21.15.163:8765 2024-06-18T03:04:00.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:59 smithi163 ceph-mon[25094]: [18/Jun/2024:03:03:58] ENGINE Bus STARTED 2024-06-18T03:04:00.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:59 smithi163 ceph-mon[25094]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:04:00.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:59 smithi163 ceph-mon[25094]: mgrmap e38: y(active, since 3s), standbys: x 2024-06-18T03:04:00.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:03:59 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:00.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:59 smithi163 ceph-mon[29791]: [18/Jun/2024:03:03:58] ENGINE Serving on http://172.21.15.163:8765 2024-06-18T03:04:00.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:59 smithi163 ceph-mon[29791]: [18/Jun/2024:03:03:58] ENGINE Bus STARTED 2024-06-18T03:04:00.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:59 smithi163 ceph-mon[29791]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:04:00.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:59 smithi163 ceph-mon[29791]: mgrmap e38: y(active, since 3s), standbys: x 2024-06-18T03:04:00.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:03:59 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:00.298 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:04:00.228+0000 7f9cc1cd6040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T03:04:00.298 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:59 smithi186 ceph-mon[25944]: [18/Jun/2024:03:03:58] ENGINE Serving on http://172.21.15.163:8765 2024-06-18T03:04:00.298 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:59 smithi186 ceph-mon[25944]: [18/Jun/2024:03:03:58] ENGINE Bus STARTED 2024-06-18T03:04:00.298 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:59 smithi186 ceph-mon[25944]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:04:00.298 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:59 smithi186 ceph-mon[25944]: mgrmap e38: y(active, since 3s), standbys: x 2024-06-18T03:04:00.298 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:03:59 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:00.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:04:00.297+0000 7f9cc1cd6040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T03:04:00.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:04:00.473+0000 7f9cc1cd6040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-18T03:04:00.875 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:04:00.801+0000 7f9cc1cd6040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T03:04:01.234 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:04:00.874+0000 7f9cc1cd6040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T03:04:01.234 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:04:01.088+0000 7f9cc1cd6040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T03:04:01.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:04:01.266+0000 7f9cc1cd6040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T03:04:01.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:04:01.333+0000 7f9cc1cd6040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T03:04:01.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:04:01.421+0000 7f9cc1cd6040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T03:04:01.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:04:01.482+0000 7f9cc1cd6040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T03:04:02.365 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:04:02.082+0000 7f9cc1cd6040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T03:04:02.365 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:04:02.257+0000 7f9cc1cd6040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T03:04:02.365 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:03:04:02] ENGINE Bus STARTING 2024-06-18T03:04:02.365 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: CherryPy Checker: 2024-06-18T03:04:02.365 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: The Application mounted at '' has an empty config. 2024-06-18T03:04:02.365 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: 2024-06-18T03:04:02.365 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:04:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-18T03:04:02.365 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:03:04:02] ENGINE Serving on http://:::9283 2024-06-18T03:04:02.365 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[45511]: [18/Jun/2024:03:04:02] ENGINE Bus STARTED 2024-06-18T03:04:02.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:04:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: mgrmap e39: y(active, since 5s), standbys: x 2024-06-18T03:04:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:04:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:04:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: Standby manager daemon x restarted 2024-06-18T03:04:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: Standby manager daemon x started 2024-06-18T03:04:02.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/20892208' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T03:04:02.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/20892208' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T03:04:02.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/20892208' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T03:04:02.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/20892208' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T03:04:02.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:04:02.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: mgrmap e39: y(active, since 5s), standbys: x 2024-06-18T03:04:02.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:04:02.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:04:02.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: Standby manager daemon x restarted 2024-06-18T03:04:02.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: Standby manager daemon x started 2024-06-18T03:04:02.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/20892208' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T03:04:02.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/20892208' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T03:04:02.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/20892208' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T03:04:02.799 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:02 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/20892208' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T03:04:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:04:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: mgrmap e39: y(active, since 5s), standbys: x 2024-06-18T03:04:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:02.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:04:02.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:04:02.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: Standby manager daemon x restarted 2024-06-18T03:04:02.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: Standby manager daemon x started 2024-06-18T03:04:02.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/20892208' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T03:04:02.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/20892208' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T03:04:02.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/20892208' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T03:04:02.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:02 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/20892208' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T03:04:03.791 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:03 smithi186 ceph-mon[25944]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T03:04:03.791 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:03 smithi186 ceph-mon[25944]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T03:04:03.791 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:03 smithi186 ceph-mon[25944]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:04:03.791 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:03 smithi186 ceph-mon[25944]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:04:03.791 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:03 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.791 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:03 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.791 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:03 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.791 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:03 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.791 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:03 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.791 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:03 smithi186 ceph-mon[25944]: mgrmap e40: y(active, since 7s), standbys: x 2024-06-18T03:04:03.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[29791]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T03:04:03.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[29791]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T03:04:03.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[29791]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:04:03.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[29791]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:04:03.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[29791]: mgrmap e40: y(active, since 7s), standbys: x 2024-06-18T03:04:03.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[25094]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T03:04:03.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[25094]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T03:04:03.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[25094]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:04:03.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[25094]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:04:03.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:03.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:03 smithi163 ceph-mon[25094]: mgrmap e40: y(active, since 7s), standbys: x 2024-06-18T03:04:04.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[29791]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:04:04.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[29791]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:04:04.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[29791]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:04:04.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[29791]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:04:04.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[29791]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:04:04.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[29791]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 0 B/s wr, 179 op/s 2024-06-18T03:04:04.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[29791]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T03:04:04.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[29791]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T03:04:04.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[29791]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-06-18T03:04:04.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[25094]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:04:04.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[25094]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:04:04.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[25094]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:04:04.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[25094]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:04:04.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[25094]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:04:04.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[25094]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 0 B/s wr, 179 op/s 2024-06-18T03:04:04.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[25094]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T03:04:04.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[25094]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T03:04:04.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:04 smithi163 ceph-mon[25094]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-06-18T03:04:04.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:04 smithi186 ceph-mon[25944]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:04:04.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:04 smithi186 ceph-mon[25944]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:04:04.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:04 smithi186 ceph-mon[25944]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:04:04.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:04 smithi186 ceph-mon[25944]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:04:04.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:04 smithi186 ceph-mon[25944]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:04:04.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:04 smithi186 ceph-mon[25944]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 0 B/s wr, 179 op/s 2024-06-18T03:04:04.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:04 smithi186 ceph-mon[25944]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-18T03:04:04.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:04 smithi186 ceph-mon[25944]: Reconfiguring daemon prometheus.a on smithi186 2024-06-18T03:04:04.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:04 smithi186 ceph-mon[25944]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-06-18T03:04:06.566 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 systemd[1]: Stopping Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:04:06.566 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 bash[60208]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-18T03:04:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:06 smithi186 ceph-mon[25944]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 128 op/s 2024-06-18T03:04:06.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:04:06.597Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-18T03:04:06.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:04:06.597Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-06-18T03:04:06.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:04:06.597Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-06-18T03:04:06.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:04:06.597Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-06-18T03:04:06.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:04:06.597Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-06-18T03:04:06.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:04:06.597Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-06-18T03:04:06.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:04:06.597Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-18T03:04:06.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:04:06.597Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-06-18T03:04:06.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:04:06.597Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-06-18T03:04:06.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:04:06.597Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-06-18T03:04:06.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:04:06.597Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-06-18T03:04:06.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[57482]: ts=2024-06-18T03:04:06.598Z caller=main.go:1066 level=info msg="See you next time!" 2024-06-18T03:04:06.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 podman[60218]: 2024-06-18 03:04:06.607547693 +0000 UTC m=+0.152023696 container died 158474d404977558284e1da95c0acdee3e3effdf7d81fa93f24dce302d94e128 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:07.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:06 smithi163 ceph-mon[29791]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 128 op/s 2024-06-18T03:04:07.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:06 smithi163 ceph-mon[25094]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 0 B/s wr, 128 op/s 2024-06-18T03:04:07.230 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 podman[60218]: 2024-06-18 03:04:06.988377139 +0000 UTC m=+0.532853142 container cleanup 158474d404977558284e1da95c0acdee3e3effdf7d81fa93f24dce302d94e128 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:07.231 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:06 smithi186 bash[60218]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a 2024-06-18T03:04:07.231 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 bash[60247]: Error: no container with name or ID "ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-18T03:04:07.481 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 podman[60232]: 2024-06-18 03:04:07.231175922 +0000 UTC m=+0.625166224 container remove 158474d404977558284e1da95c0acdee3e3effdf7d81fa93f24dce302d94e128 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:07.482 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-06-18T03:04:07.482 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 systemd[1]: Stopped Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:04:07.482 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a.service: Consumed 1.415s CPU time. 2024-06-18T03:04:07.482 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 systemd[1]: Starting Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:04:07.482 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 podman[60310]: 2024-06-18 03:04:07.37536828 +0000 UTC m=+0.020345200 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-06-18T03:04:07.750 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 podman[60310]: 2024-06-18 03:04:07.482190703 +0000 UTC m=+0.127167632 container create 1ac8cbed34dea75d4c73a425a2fe438428b9b6d55a06ad5c4a9aa6cb59afe633 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:08.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 podman[60310]: 2024-06-18 03:04:07.751409504 +0000 UTC m=+0.396386425 container init 1ac8cbed34dea75d4c73a425a2fe438428b9b6d55a06ad5c4a9aa6cb59afe633 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:08.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 podman[60310]: 2024-06-18 03:04:07.754357928 +0000 UTC m=+0.399334848 container start 1ac8cbed34dea75d4c73a425a2fe438428b9b6d55a06ad5c4a9aa6cb59afe633 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:08.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:07.778Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-06-18T03:04:08.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:07.778Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-06-18T03:04:08.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:07.778Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-06-18T03:04:08.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:07.779Z caller=main.go:518 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi186 (none))" 2024-06-18T03:04:08.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:07.779Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-18T03:04:08.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:07.779Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-18T03:04:08.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:07.780Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-06-18T03:04:08.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:07.780Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-06-18T03:04:08.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:07.781Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-06-18T03:04:08.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:07.783Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-18T03:04:08.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:07.783Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=9.514µs 2024-06-18T03:04:08.070 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:07.783Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-18T03:04:08.070 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 bash[60310]: 1ac8cbed34dea75d4c73a425a2fe438428b9b6d55a06ad5c4a9aa6cb59afe633 2024-06-18T03:04:08.070 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:07 smithi186 systemd[1]: Started Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:04:08.070 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:08.000Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-06-18T03:04:08.496 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:08.241Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-06-18T03:04:08.496 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:08.465Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-06-18T03:04:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-mon[25944]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 0 B/s wr, 100 op/s 2024-06-18T03:04:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:04:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:04:08.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:08.517Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-06-18T03:04:08.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:08.523Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-06-18T03:04:08.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:08.524Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-06-18T03:04:08.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:08.524Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=70.339µs wal_replay_duration=740.331117ms total_replay_duration=740.477131ms 2024-06-18T03:04:08.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:08.528Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-06-18T03:04:08.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:08.528Z caller=main.go:947 level=info msg="TSDB started" 2024-06-18T03:04:08.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:08.528Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-18T03:04:08.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:08.547Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.343891ms db_storage=860ns remote_storage=1.644µs web_handler=339ns query_engine=738ns scrape=501.308µs scrape_sd=99.758µs notify=15.148µs notify_sd=17.013µs rules=18.296289ms 2024-06-18T03:04:08.819 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:08.548Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-06-18T03:04:09.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:08 smithi163 ceph-mon[25094]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 0 B/s wr, 100 op/s 2024-06-18T03:04:09.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:08 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:09.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:08 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:09.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:08 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:04:09.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:08 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:04:09.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:08 smithi163 ceph-mon[29791]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 0 B/s wr, 100 op/s 2024-06-18T03:04:09.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:08 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:09.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:08 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:09.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:08 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:04:09.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:08 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:04:09.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:09 smithi186 ceph-mon[25944]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:04:09.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:09 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T03:04:09.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:09 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T03:04:09.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:09 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:04:10.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:09 smithi163 ceph-mon[25094]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:04:10.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:09 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T03:04:10.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:09 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T03:04:10.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:09 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:04:10.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:09 smithi163 ceph-mon[29791]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:04:10.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:09 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T03:04:10.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:09 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T03:04:10.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:09 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:04:10.742 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:10 smithi186 ceph-mon[25944]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 0 B/s wr, 153 op/s 2024-06-18T03:04:10.742 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:10 smithi186 ceph-mon[25944]: Upgrade: Updating mgr.x 2024-06-18T03:04:10.743 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:10 smithi186 ceph-mon[25944]: Deploying daemon mgr.x on smithi186 2024-06-18T03:04:10.743 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:10 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:11.042 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:10 smithi186 systemd[1]: Stopping Ceph mgr.x for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:04:11.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:10 smithi163 ceph-mon[25094]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 0 B/s wr, 153 op/s 2024-06-18T03:04:11.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:10 smithi163 ceph-mon[25094]: Upgrade: Updating mgr.x 2024-06-18T03:04:11.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:10 smithi163 ceph-mon[25094]: Deploying daemon mgr.x on smithi186 2024-06-18T03:04:11.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:10 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:11.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:10 smithi163 ceph-mon[29791]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 0 B/s wr, 153 op/s 2024-06-18T03:04:11.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:10 smithi163 ceph-mon[29791]: Upgrade: Updating mgr.x 2024-06-18T03:04:11.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:10 smithi163 ceph-mon[29791]: Deploying daemon mgr.x on smithi186 2024-06-18T03:04:11.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:10 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:11.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:11 smithi186 podman[60655]: 2024-06-18 03:04:11.041944448 +0000 UTC m=+0.153404996 container died d7bd16a633ffbf9fb48e1b7e707070457044f9d33f468e61f20ed46867637863 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, ceph=True, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-06-18T03:04:11.771 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:11 smithi186 podman[60655]: 2024-06-18 03:04:11.469932792 +0000 UTC m=+0.581393334 container cleanup d7bd16a633ffbf9fb48e1b7e707070457044f9d33f468e61f20ed46867637863 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-06-18T03:04:11.771 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:11 smithi186 bash[60655]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x 2024-06-18T03:04:12.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:11 smithi186 podman[60669]: 2024-06-18 03:04:11.771505071 +0000 UTC m=+0.726371372 container remove d7bd16a633ffbf9fb48e1b7e707070457044f9d33f468e61f20ed46867637863 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, RELEASE=squid-3781e3a, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True) 2024-06-18T03:04:12.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:11 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-06-18T03:04:12.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:11 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-06-18T03:04:12.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:11 smithi186 systemd[1]: Stopped Ceph mgr.x for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:04:12.068 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:11 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.x.service: Consumed 31.805s CPU time. 2024-06-18T03:04:12.423 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:12 smithi186 systemd[1]: Starting Ceph mgr.x for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:04:12.423 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:12 smithi186 podman[60798]: 2024-06-18 03:04:12.313985535 +0000 UTC m=+0.112060100 container create cf365ee8b674129940913305ed6e62988ebd190a3a4fe27e13a2dc3fb11e9691 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, RELEASE=squid-3781e3a, org.label-schema.build-date=20240612, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-06-18T03:04:12.423 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:12 smithi186 podman[60798]: 2024-06-18 03:04:12.21960775 +0000 UTC m=+0.017682314 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:04:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:12 smithi186 ceph-mon[25944]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 0 B/s wr, 153 op/s 2024-06-18T03:04:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:12 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:12 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:04:12.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:12 smithi186 podman[60798]: 2024-06-18 03:04:12.533434219 +0000 UTC m=+0.331508791 container init cf365ee8b674129940913305ed6e62988ebd190a3a4fe27e13a2dc3fb11e9691 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0) 2024-06-18T03:04:12.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:12 smithi186 podman[60798]: 2024-06-18 03:04:12.536280798 +0000 UTC m=+0.334355356 container start cf365ee8b674129940913305ed6e62988ebd190a3a4fe27e13a2dc3fb11e9691 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-06-18T03:04:12.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:12 smithi186 ceph-mgr[60816]: -- 172.21.15.186:0/3280859452 <== mon.2 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d6090970e0 con 0x55d609075400 2024-06-18T03:04:12.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:12 smithi186 bash[60798]: cf365ee8b674129940913305ed6e62988ebd190a3a4fe27e13a2dc3fb11e9691 2024-06-18T03:04:12.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:12 smithi186 systemd[1]: Started Ceph mgr.x for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:04:12.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:12.666+0000 7f6984077040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T03:04:13.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:12 smithi163 ceph-mon[29791]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 0 B/s wr, 153 op/s 2024-06-18T03:04:13.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:12 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:13.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:12 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:04:13.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:12 smithi163 ceph-mon[25094]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 0 B/s wr, 153 op/s 2024-06-18T03:04:13.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:12 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:13.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:12 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:04:13.185 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:12.831+0000 7f6984077040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T03:04:13.186 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:12.904+0000 7f6984077040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T03:04:13.186 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:13.185+0000 7f6984077040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T03:04:13.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:13.245+0000 7f6984077040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T03:04:13.989 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:13 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:13.989 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:13 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:13.990 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:13 smithi186 ceph-mon[25944]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 0 B/s wr, 154 op/s 2024-06-18T03:04:14.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:13 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:14.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:13 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:14.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:13 smithi163 ceph-mon[25094]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 0 B/s wr, 154 op/s 2024-06-18T03:04:14.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:04:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:04:13] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.33.4" 2024-06-18T03:04:14.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:13 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:14.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:13 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:14.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:13 smithi163 ceph-mon[29791]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 0 B/s wr, 154 op/s 2024-06-18T03:04:14.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:14.266+0000 7f6984077040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T03:04:14.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:14.385+0000 7f6984077040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T03:04:14.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:14.447+0000 7f6984077040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T03:04:14.990 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:14.700+0000 7f6984077040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T03:04:14.991 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:14.764+0000 7f6984077040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T03:04:14.991 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:14.832+0000 7f6984077040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T03:04:14.991 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:14.990+0000 7f6984077040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T03:04:15.737 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:15.437+0000 7f6984077040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T03:04:15.998 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:15.774+0000 7f6984077040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T03:04:15.998 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:15.843+0000 7f6984077040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T03:04:15.998 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:15.997+0000 7f6984077040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-18T03:04:16.566 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:16 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:16.334+0000 7f6984077040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T03:04:16.566 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:16 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:16.399+0000 7f6984077040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T03:04:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:16 smithi186 ceph-mon[25944]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 53 op/s 2024-06-18T03:04:16.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:16 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:16.591+0000 7f6984077040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T03:04:16.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:16 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:16.749+0000 7f6984077040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T03:04:17.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:16 smithi163 ceph-mon[29791]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 53 op/s 2024-06-18T03:04:17.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:16 smithi163 ceph-mon[25094]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 53 op/s 2024-06-18T03:04:17.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:16 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:16.820+0000 7f6984077040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T03:04:17.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:16 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:16.911+0000 7f6984077040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T03:04:17.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:16 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:16.974+0000 7f6984077040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T03:04:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-mon[25944]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 53 op/s 2024-06-18T03:04:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:17.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-mon[25944]: Standby manager daemon x restarted 2024-06-18T03:04:17.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-mon[25944]: Standby manager daemon x started 2024-06-18T03:04:17.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/462557238' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T03:04:17.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/462557238' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T03:04:17.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/462557238' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T03:04:17.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-mon[25944]: from='mgr.? 172.21.15.186:0/462557238' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T03:04:17.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:17.561+0000 7f6984077040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T03:04:17.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:17.745+0000 7f6984077040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T03:04:17.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: [18/Jun/2024:03:04:17] ENGINE Bus STARTING 2024-06-18T03:04:17.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: CherryPy Checker: 2024-06-18T03:04:17.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: The Application mounted at '' has an empty config. 2024-06-18T03:04:17.819 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:04:18.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:18.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:18.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[29791]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 53 op/s 2024-06-18T03:04:18.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:18.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:18.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[29791]: Standby manager daemon x restarted 2024-06-18T03:04:18.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[29791]: Standby manager daemon x started 2024-06-18T03:04:18.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/462557238' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T03:04:18.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/462557238' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T03:04:18.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/462557238' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T03:04:18.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/462557238' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T03:04:18.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:18.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:18.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[25094]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 53 op/s 2024-06-18T03:04:18.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:18.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:18.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[25094]: Standby manager daemon x restarted 2024-06-18T03:04:18.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[25094]: Standby manager daemon x started 2024-06-18T03:04:18.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/462557238' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T03:04:18.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/462557238' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T03:04:18.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/462557238' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T03:04:18.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:17 smithi163 ceph-mon[25094]: from='mgr.? 172.21.15.186:0/462557238' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T03:04:18.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: [18/Jun/2024:03:04:17] ENGINE Serving on http://:::9283 2024-06-18T03:04:18.073 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:04:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: [18/Jun/2024:03:04:17] ENGINE Bus STARTED 2024-06-18T03:04:19.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:19 smithi163 ceph-mon[29791]: mgrmap e41: y(active, since 22s), standbys: x 2024-06-18T03:04:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:19 smithi163 ceph-mon[25094]: mgrmap e41: y(active, since 22s), standbys: x 2024-06-18T03:04:19.816 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:19 smithi186 ceph-mon[25944]: mgrmap e41: y(active, since 22s), standbys: x 2024-06-18T03:04:20.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:20 smithi186 ceph-mon[25944]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 54 op/s 2024-06-18T03:04:20.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:20 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:20.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:20 smithi163 ceph-mon[29791]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 54 op/s 2024-06-18T03:04:20.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:20 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:20 smithi163 ceph-mon[25094]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 54 op/s 2024-06-18T03:04:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:20 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 905 B/s rd, 0 op/s 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all mgr 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all crash 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all mds 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all nfs 2024-06-18T03:04:21.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:04:21.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:04:21.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 905 B/s rd, 0 op/s 2024-06-18T03:04:21.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:21.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all mgr 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all crash 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all mds 2024-06-18T03:04:21.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all nfs 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:04:21.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:04:21.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:04:21.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:04:21.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 905 B/s rd, 0 op/s 2024-06-18T03:04:21.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:21.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:04:21.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: Upgrade: Setting container_image for all mgr 2024-06-18T03:04:21.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-06-18T03:04:21.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-06-18T03:04:21.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-06-18T03:04:21.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-06-18T03:04:21.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: Upgrade: Setting container_image for all crash 2024-06-18T03:04:21.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: Upgrade: Setting container_image for all mds 2024-06-18T03:04:21.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:04:21.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-18T03:04:21.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:04:21.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: Upgrade: Setting container_image for all nfs 2024-06-18T03:04:21.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:21.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:21.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:04:21.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:22.543 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:22 smithi163 systemd[1]: Stopping Ceph node-exporter.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:04:22.938 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:22 smithi163 podman[75973]: 2024-06-18 03:04:22.641969417 +0000 UTC m=+0.125865583 container died 8d0aecfcef8a83349c1cdc0291918331c3c0ea8c24f66f604ee3599aa9e25f5f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:22.938 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:22 smithi163 podman[75973]: 2024-06-18 03:04:22.939227589 +0000 UTC m=+0.423123754 container cleanup 8d0aecfcef8a83349c1cdc0291918331c3c0ea8c24f66f604ee3599aa9e25f5f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:23.292 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:22 smithi163 bash[75973]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a 2024-06-18T03:04:23.292 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:23 smithi163 podman[75988]: 2024-06-18 03:04:23.173909475 +0000 UTC m=+0.530355493 container remove 8d0aecfcef8a83349c1cdc0291918331c3c0ea8c24f66f604ee3599aa9e25f5f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:23.293 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:23 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-06-18T03:04:23.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:23 smithi163 ceph-mon[29791]: Upgrade: Updating node-exporter.a (1/2) 2024-06-18T03:04:23.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:23 smithi163 ceph-mon[29791]: Deploying daemon node-exporter.a on smithi163 2024-06-18T03:04:23.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:23 smithi163 ceph-mon[29791]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 550 B/s rd, 0 op/s 2024-06-18T03:04:23.710 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:23 smithi163 ceph-mon[25094]: Upgrade: Updating node-exporter.a (1/2) 2024-06-18T03:04:23.710 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:23 smithi163 ceph-mon[25094]: Deploying daemon node-exporter.a on smithi163 2024-06-18T03:04:23.711 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:23 smithi163 ceph-mon[25094]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 550 B/s rd, 0 op/s 2024-06-18T03:04:23.711 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:23 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-06-18T03:04:23.711 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:23 smithi163 systemd[1]: Stopped Ceph node-exporter.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:04:23.711 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:23 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.a.service: Consumed 5.146s CPU time. 2024-06-18T03:04:23.711 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:23 smithi163 systemd[1]: Starting Ceph node-exporter.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:04:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:23 smithi186 ceph-mon[25944]: Upgrade: Updating node-exporter.a (1/2) 2024-06-18T03:04:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:23 smithi186 ceph-mon[25944]: Deploying daemon node-exporter.a on smithi163 2024-06-18T03:04:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:23 smithi186 ceph-mon[25944]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 550 B/s rd, 0 op/s 2024-06-18T03:04:24.042 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:23 smithi163 bash[76117]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-06-18T03:04:24.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:04:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:04:23] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.33.4" 2024-06-18T03:04:25.292 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:24 smithi163 bash[76117]: Getting image source signatures 2024-06-18T03:04:25.293 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:24 smithi163 bash[76117]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-06-18T03:04:25.293 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:24 smithi163 bash[76117]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-06-18T03:04:25.293 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:24 smithi163 bash[76117]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-06-18T03:04:25.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:25 smithi163 ceph-mon[25094]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:25.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:25 smithi163 ceph-mon[29791]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:25 smithi186 ceph-mon[25944]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:26.792 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:26 smithi163 bash[76117]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-06-18T03:04:26.792 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:26 smithi163 bash[76117]: Writing manifest to image destination 2024-06-18T03:04:26.913 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:04:27.103 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:26 smithi163 podman[76117]: 2024-06-18 03:04:26.760842156 +0000 UTC m=+3.144623644 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-06-18T03:04:27.104 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:26 smithi163 podman[76117]: 2024-06-18 03:04:26.876075393 +0000 UTC m=+3.259856873 container create 6b0b89b4bc8567b6b4b05ab39e6a6580b34ae4d089cb4d479682c290dc85693f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:27.104 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 podman[76117]: 2024-06-18 03:04:27.103526962 +0000 UTC m=+3.487308450 container init 6b0b89b4bc8567b6b4b05ab39e6a6580b34ae4d089cb4d479682c290dc85693f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:27.463 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 podman[76117]: 2024-06-18 03:04:27.108041896 +0000 UTC m=+3.491823383 container start 6b0b89b4bc8567b6b4b05ab39e6a6580b34ae4d089cb4d479682c290dc85693f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:27.463 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.112Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-06-18T03:04:27.463 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.112Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-06-18T03:04:27.463 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.113Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-06-18T03:04:27.463 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.113Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-06-18T03:04:27.463 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-06-18T03:04:27.463 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-06-18T03:04:27.463 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-06-18T03:04:27.463 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=arp 2024-06-18T03:04:27.463 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=bcache 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=bonding 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=btrfs 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=conntrack 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=cpu 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=diskstats 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=dmi 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=edac 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=entropy 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=filefd 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=filesystem 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=hwmon 2024-06-18T03:04:27.464 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=infiniband 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=ipvs 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=loadavg 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=mdadm 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=meminfo 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=netclass 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=netdev 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=netstat 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=nfs 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=nfsd 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=nvme 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.114Z caller=node_exporter.go:117 level=info collector=os 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=pressure 2024-06-18T03:04:27.465 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=rapl 2024-06-18T03:04:27.466 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=schedstat 2024-06-18T03:04:27.466 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=selinux 2024-06-18T03:04:27.466 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=sockstat 2024-06-18T03:04:27.466 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=softnet 2024-06-18T03:04:27.466 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=stat 2024-06-18T03:04:27.466 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=tapestats 2024-06-18T03:04:27.466 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=textfile 2024-06-18T03:04:27.466 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-06-18T03:04:27.466 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=time 2024-06-18T03:04:27.467 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-06-18T03:04:27.467 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=uname 2024-06-18T03:04:27.467 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=vmstat 2024-06-18T03:04:27.467 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=xfs 2024-06-18T03:04:27.467 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=node_exporter.go:117 level=info collector=zfs 2024-06-18T03:04:27.467 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-06-18T03:04:27.467 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a[76212]: ts=2024-06-18T03:04:27.115Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-06-18T03:04:27.467 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 bash[76117]: 6b0b89b4bc8567b6b4b05ab39e6a6580b34ae4d089cb4d479682c290dc85693f 2024-06-18T03:04:27.467 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:04:27 smithi163 systemd[1]: Started Ceph node-exporter.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:04:27.587 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:04:27.587 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (13m) 26s ago 21m 21.7M - 0.23.0 ba2b418f427c 5e109a483ef4 2024-06-18T03:04:27.587 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (13m) 10s ago 20m 53.4M - 8.3.5 dad864ee21e9 a3fd9ade8374 2024-06-18T03:04:27.587 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 unknown 26s ago 20m 50.8M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:04:27.587 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (15s) 10s ago 25m 53.3M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:04:27.588 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (9m) 26s ago 27m 475M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:04:27.588 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (27m) 26s ago 27m 79.4M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T03:04:27.588 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (25m) 10s ago 25m 57.6M 2048M 17.2.0 e1d6a67b021e a3618f734a9f 2024-06-18T03:04:27.588 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (26m) 26s ago 26m 55.2M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T03:04:27.588 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 starting - - - - 2024-06-18T03:04:27.588 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (13m) 10s ago 21m 17.2M - 1.3.1 1dbe0e931976 0592fede9474 2024-06-18T03:04:27.588 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (25m) 26s ago 25m 55.2M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T03:04:27.588 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (24m) 26s ago 24m 55.1M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T03:04:27.588 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (24m) 26s ago 24m 50.7M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:04:27.588 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (24m) 26s ago 24m 53.7M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:04:27.588 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (23m) 10s ago 23m 61.6M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:04:27.588 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (23m) 10s ago 23m 52.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:04:27.589 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (23m) 10s ago 23m 51.8M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:04:27.589 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (22m) 10s ago 22m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:04:27.589 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (20s) 10s ago 21m 50.7M - 2.33.4 514e6a882f6e 1ac8cbed34de 2024-06-18T03:04:27.589 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (20m) 26s ago 20m 78.6M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:04:27.589 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (20m) 10s ago 20m 79.5M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:04:27.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-mon[29791]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:27.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:04:27.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:27.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:27.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-mon[25094]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:27.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:04:27.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:27.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:27 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:27.903 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:27 smithi186 ceph-mon[25944]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:27.903 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:27 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:04:27.903 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:27 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:27.903 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:27 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:28.014 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:04:28.014 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: "mds": {}, 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:04:28.015 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-18T03:04:28.016 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:04:28.016 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:04:28.016 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:04:28.348 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:04:28.348 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:04:28.349 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:04:28.349 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-06-18T03:04:28.349 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [ 2024-06-18T03:04:28.349 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr" 2024-06-18T03:04:28.349 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2024-06-18T03:04:28.349 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "2/2 daemons upgraded", 2024-06-18T03:04:28.349 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading node-exporter daemons", 2024-06-18T03:04:28.349 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:04:28.349 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:04:28.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:28 smithi163 ceph-mon[29791]: from='client.25402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:04:28.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:28 smithi163 ceph-mon[29791]: from='client.15504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:04:28.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:28 smithi163 ceph-mon[29791]: from='client.15510 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:04:28.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:28 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1690480488' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:28 smithi163 ceph-mon[25094]: from='client.25402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:04:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:28 smithi163 ceph-mon[25094]: from='client.15504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:04:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:28 smithi163 ceph-mon[25094]: from='client.15510 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:04:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:28 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1690480488' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:28 smithi186 ceph-mon[25944]: from='client.25402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:04:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:28 smithi186 ceph-mon[25944]: from='client.15504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:04:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:28 smithi186 ceph-mon[25944]: from='client.15510 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:04:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:28 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1690480488' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:29.318 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:29 smithi186 systemd[1]: Stopping Ceph node-exporter.b for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:04:29.575 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:29 smithi186 podman[62132]: 2024-06-18 03:04:29.321914736 +0000 UTC m=+0.114770345 container died 0592fede9474f20dd00e22f4bfe87cb492f4879ef1567fa785bdcf9b4249e59c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T03:04:30.021 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:29 smithi186 ceph-mon[25944]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:04:30.021 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:29 smithi186 ceph-mon[25944]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:30.021 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:29 smithi186 ceph-mon[25944]: Upgrade: Updating node-exporter.b (2/2) 2024-06-18T03:04:30.021 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:29 smithi186 ceph-mon[25944]: Deploying daemon node-exporter.b on smithi186 2024-06-18T03:04:30.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:29 smithi186 podman[62132]: 2024-06-18 03:04:29.644912673 +0000 UTC m=+0.437768272 container cleanup 0592fede9474f20dd00e22f4bfe87cb492f4879ef1567fa785bdcf9b4249e59c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T03:04:30.021 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:29 smithi186 bash[62132]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b 2024-06-18T03:04:30.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:29 smithi163 ceph-mon[29791]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:04:30.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:29 smithi163 ceph-mon[29791]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:30.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:29 smithi163 ceph-mon[29791]: Upgrade: Updating node-exporter.b (2/2) 2024-06-18T03:04:30.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:29 smithi163 ceph-mon[29791]: Deploying daemon node-exporter.b on smithi186 2024-06-18T03:04:30.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:29 smithi163 ceph-mon[25094]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:04:30.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:29 smithi163 ceph-mon[25094]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:30.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:29 smithi163 ceph-mon[25094]: Upgrade: Updating node-exporter.b (2/2) 2024-06-18T03:04:30.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:29 smithi163 ceph-mon[25094]: Deploying daemon node-exporter.b on smithi186 2024-06-18T03:04:30.296 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:30 smithi186 podman[62144]: 2024-06-18 03:04:30.021246353 +0000 UTC m=+0.702867614 container remove 0592fede9474f20dd00e22f4bfe87cb492f4879ef1567fa785bdcf9b4249e59c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T03:04:30.296 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:30 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-06-18T03:04:30.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:30 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-06-18T03:04:30.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:30 smithi186 systemd[1]: Stopped Ceph node-exporter.b for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:04:30.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:30 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.b.service: Consumed 5.341s CPU time. 2024-06-18T03:04:30.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:30 smithi186 systemd[1]: Starting Ceph node-exporter.b for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:04:31.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:30 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:31.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:30 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:31.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:30 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:31.068 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:30 smithi186 bash[62273]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-06-18T03:04:31.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:31 smithi186 bash[62273]: Getting image source signatures 2024-06-18T03:04:31.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:31 smithi186 bash[62273]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-06-18T03:04:31.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:31 smithi186 bash[62273]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-06-18T03:04:31.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:31 smithi186 bash[62273]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-06-18T03:04:32.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:31 smithi163 ceph-mon[29791]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:04:32.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:31 smithi163 ceph-mon[25094]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:04:32.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:31 smithi186 ceph-mon[25944]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:04:32.818 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:32 smithi186 bash[62273]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-06-18T03:04:32.818 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:32 smithi186 bash[62273]: Writing manifest to image destination 2024-06-18T03:04:33.097 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:32 smithi186 podman[62273]: 2024-06-18 03:04:32.834698802 +0000 UTC m=+2.220507916 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-06-18T03:04:33.097 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:32 smithi186 podman[62273]: 2024-06-18 03:04:32.976832467 +0000 UTC m=+2.362641623 container create 8a17d71d6acb9520cf6e136cd705fe45e9fdc7414dad15a2bf12476af8316a5a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T03:04:33.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 podman[62273]: 2024-06-18 03:04:33.202219917 +0000 UTC m=+2.588029038 container init 8a17d71d6acb9520cf6e136cd705fe45e9fdc7414dad15a2bf12476af8316a5a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T03:04:33.568 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 podman[62273]: 2024-06-18 03:04:33.205175705 +0000 UTC m=+2.590984829 container start 8a17d71d6acb9520cf6e136cd705fe45e9fdc7414dad15a2bf12476af8316a5a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T03:04:33.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.209Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-06-18T03:04:33.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.209Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-06-18T03:04:33.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.209Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-06-18T03:04:33.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.209Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-06-18T03:04:33.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-06-18T03:04:33.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-06-18T03:04:33.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-06-18T03:04:33.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=arp 2024-06-18T03:04:33.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=bcache 2024-06-18T03:04:33.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=bonding 2024-06-18T03:04:33.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=btrfs 2024-06-18T03:04:33.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=conntrack 2024-06-18T03:04:33.570 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=cpu 2024-06-18T03:04:33.570 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-06-18T03:04:33.570 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=diskstats 2024-06-18T03:04:33.570 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=dmi 2024-06-18T03:04:33.570 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=edac 2024-06-18T03:04:33.570 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=entropy 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=filefd 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=filesystem 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=hwmon 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=infiniband 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=ipvs 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=loadavg 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=mdadm 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=meminfo 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=netclass 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=netdev 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=netstat 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=nfs 2024-06-18T03:04:33.571 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=nfsd 2024-06-18T03:04:33.572 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=nvme 2024-06-18T03:04:33.572 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=os 2024-06-18T03:04:33.572 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-06-18T03:04:33.572 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=pressure 2024-06-18T03:04:33.572 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=rapl 2024-06-18T03:04:33.572 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=schedstat 2024-06-18T03:04:33.572 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=selinux 2024-06-18T03:04:33.572 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=sockstat 2024-06-18T03:04:33.572 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=softnet 2024-06-18T03:04:33.572 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=stat 2024-06-18T03:04:33.572 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=tapestats 2024-06-18T03:04:33.572 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=textfile 2024-06-18T03:04:33.573 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-06-18T03:04:33.573 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=time 2024-06-18T03:04:33.573 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-06-18T03:04:33.573 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=uname 2024-06-18T03:04:33.573 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=vmstat 2024-06-18T03:04:33.573 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=xfs 2024-06-18T03:04:33.573 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.210Z caller=node_exporter.go:117 level=info collector=zfs 2024-06-18T03:04:33.573 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.211Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-06-18T03:04:33.573 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b[62322]: ts=2024-06-18T03:04:33.211Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-06-18T03:04:33.573 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 bash[62273]: 8a17d71d6acb9520cf6e136cd705fe45e9fdc7414dad15a2bf12476af8316a5a 2024-06-18T03:04:33.573 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:04:33 smithi186 systemd[1]: Started Ceph node-exporter.b for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:04:33.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:33 smithi163 ceph-mon[29791]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:33.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:33 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:33.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:33 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:33.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:33 smithi163 ceph-mon[25094]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:33.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:33 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:33.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:33 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:33.974 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-mon[25944]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:33.974 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:33.974 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:33 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:34.130 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:04:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:04:33] "GET /metrics HTTP/1.1" 200 37995 "" "Prometheus/2.33.4" 2024-06-18T03:04:35.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:35 smithi186 ceph-mon[25944]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:04:36.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:35 smithi163 ceph-mon[29791]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:04:36.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:35 smithi163 ceph-mon[25094]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:04:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:37 smithi186 ceph-mon[25944]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:37 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:37 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:37.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:37 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:37.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:37 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:37.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:37 smithi163 ceph-mon[29791]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:37.898 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:37 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:37.898 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:37 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:37.898 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:37 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:37.898 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:37 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:37.898 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:37 smithi163 ceph-mon[25094]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:37.898 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:37 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:37.898 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:37 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:37.898 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:37 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:37.898 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:37 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:39.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:39.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:39.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:39.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:39.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:39 smithi163 ceph-mon[29791]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:04:39.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:38 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:39.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:38 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:39.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:38 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:39.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:38 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:39.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:38 smithi163 ceph-mon[25094]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:04:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:39 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:39 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:39.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:39 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:39.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:39 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:39.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:39 smithi186 ceph-mon[25944]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:04:40.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:40 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:40.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:40 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:40.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:40 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:41.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:41 smithi186 ceph-mon[25944]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:41.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:41 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:41.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:41 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:41.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:41 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:41.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:41 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:41.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:41 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:04:41.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:41 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:04:41.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:41 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:42.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[29791]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:42.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:42.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:42.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:42.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:42.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:04:42.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:04:42.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:42.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[25094]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:42.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:42.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:42.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:42.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:42.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:04:42.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:04:42.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:41 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:42.942 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:42 smithi186 ceph-mon[25944]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-06-18T03:04:42.943 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:42 smithi186 ceph-mon[25944]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:42.943 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:04:42.943 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:42.943 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:42.943 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:42.943 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:42.943 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:42.943 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:42.943 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:42.943 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:42.943 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:42.943 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:04:43.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[25094]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-06-18T03:04:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[25094]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:04:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:04:43.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[29791]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-06-18T03:04:43.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[29791]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:43.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:04:43.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:04:43.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:04:44.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:04:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:04:43] "GET /metrics HTTP/1.1" 200 37988 "" "Prometheus/2.33.4" 2024-06-18T03:04:45.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:44 smithi163 ceph-mon[25094]: Upgrade: Updating prometheus.a 2024-06-18T03:04:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:44 smithi163 ceph-mon[25094]: Deploying daemon prometheus.a on smithi186 2024-06-18T03:04:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:44 smithi163 ceph-mon[25094]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:45.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:44 smithi163 ceph-mon[29791]: Upgrade: Updating prometheus.a 2024-06-18T03:04:45.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:44 smithi163 ceph-mon[29791]: Deploying daemon prometheus.a on smithi186 2024-06-18T03:04:45.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:44 smithi163 ceph-mon[29791]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:44 smithi186 ceph-mon[25944]: Upgrade: Updating prometheus.a 2024-06-18T03:04:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:44 smithi186 ceph-mon[25944]: Deploying daemon prometheus.a on smithi186 2024-06-18T03:04:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:44 smithi186 ceph-mon[25944]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:47.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:46 smithi163 ceph-mon[29791]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:47.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:46 smithi163 ceph-mon[25094]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:47.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:46 smithi186 ceph-mon[25944]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:49.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:48 smithi163 ceph-mon[29791]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 566 B/s rd, 0 op/s 2024-06-18T03:04:49.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:48 smithi163 ceph-mon[25094]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 566 B/s rd, 0 op/s 2024-06-18T03:04:49.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:48 smithi186 ceph-mon[25944]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 566 B/s rd, 0 op/s 2024-06-18T03:04:50.892 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:50 smithi186 ceph-mon[25944]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:50.892 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:50 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:51.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:50 smithi163 ceph-mon[29791]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:51.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:50 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:51.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:50 smithi163 ceph-mon[25094]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:04:51.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:50 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:04:52.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:52 smithi186 ceph-mon[25944]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:04:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:52 smithi163 ceph-mon[29791]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:04:53.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:52 smithi163 ceph-mon[25094]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:04:53.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 systemd[1]: Stopping Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:04:54.068 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:53.753Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-18T03:04:54.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:53.753Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-06-18T03:04:54.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:53.753Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-06-18T03:04:54.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:53.753Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-06-18T03:04:54.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:53.753Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-06-18T03:04:54.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:53.753Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-06-18T03:04:54.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:53.754Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-18T03:04:54.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:53.754Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-06-18T03:04:54.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:53.754Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-06-18T03:04:54.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:53.754Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-06-18T03:04:54.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:53.754Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-06-18T03:04:54.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[60328]: ts=2024-06-18T03:04:53.754Z caller=main.go:1066 level=info msg="See you next time!" 2024-06-18T03:04:54.069 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:53 smithi186 podman[63806]: 2024-06-18 03:04:53.763934951 +0000 UTC m=+0.132202120 container died 1ac8cbed34dea75d4c73a425a2fe438428b9b6d55a06ad5c4a9aa6cb59afe633 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:54.478 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:54 smithi186 podman[63806]: 2024-06-18 03:04:54.094416803 +0000 UTC m=+0.462683969 container cleanup 1ac8cbed34dea75d4c73a425a2fe438428b9b6d55a06ad5c4a9aa6cb59afe633 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:54.479 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:54 smithi186 bash[63806]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a 2024-06-18T03:04:54.763 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:54 smithi186 ceph-mon[25944]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:04:54.763 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:54 smithi186 podman[63820]: 2024-06-18 03:04:54.47917862 +0000 UTC m=+0.710985131 container remove 1ac8cbed34dea75d4c73a425a2fe438428b9b6d55a06ad5c4a9aa6cb59afe633 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:54.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:54 smithi163 ceph-mon[29791]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:04:54.859 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:54 smithi163 ceph-mon[25094]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:04:55.044 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:54 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-06-18T03:04:55.044 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:54 smithi186 systemd[1]: Stopped Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:04:55.045 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:54 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a.service: Consumed 1.305s CPU time. 2024-06-18T03:04:55.318 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:55 smithi186 systemd[1]: Starting Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:04:55.318 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:55 smithi186 podman[63946]: 2024-06-18 03:04:55.161260659 +0000 UTC m=+0.018353084 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-06-18T03:04:56.180 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:55 smithi186 podman[63946]: 2024-06-18 03:04:55.92264877 +0000 UTC m=+0.779741224 container create fd43507a2d0669bf42d13b9a126dda2eeed1e7d8ebf60ddfe2b7a6687724ad21 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:56.180 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 podman[63946]: 2024-06-18 03:04:56.149818589 +0000 UTC m=+1.006911000 container init fd43507a2d0669bf42d13b9a126dda2eeed1e7d8ebf60ddfe2b7a6687724ad21 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:56.180 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 podman[63946]: 2024-06-18 03:04:56.153095556 +0000 UTC m=+1.010187976 container start fd43507a2d0669bf42d13b9a126dda2eeed1e7d8ebf60ddfe2b7a6687724ad21 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:04:56.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.180Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-06-18T03:04:56.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.180Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-06-18T03:04:56.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.180Z caller=main.go:570 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi186 (none))" 2024-06-18T03:04:56.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.180Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-18T03:04:56.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.180Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-18T03:04:56.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.182Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-06-18T03:04:56.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.182Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-06-18T03:04:56.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.183Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-06-18T03:04:56.569 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.183Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-06-18T03:04:56.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.184Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-18T03:04:56.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.185Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=11.819µs 2024-06-18T03:04:56.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.185Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-18T03:04:56.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.192Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-06-18T03:04:56.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.207Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-06-18T03:04:56.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.218Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-06-18T03:04:56.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.221Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-06-18T03:04:56.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.224Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-06-18T03:04:56.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.225Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-06-18T03:04:56.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.225Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-06-18T03:04:56.570 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.225Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=51.109µs wal_replay_duration=40.341642ms wbl_replay_duration=118ns total_replay_duration=40.490423ms 2024-06-18T03:04:56.571 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.229Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-06-18T03:04:56.571 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.229Z caller=main.go:1029 level=info msg="TSDB started" 2024-06-18T03:04:56.571 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.229Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-18T03:04:56.571 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 bash[63946]: fd43507a2d0669bf42d13b9a126dda2eeed1e7d8ebf60ddfe2b7a6687724ad21 2024-06-18T03:04:56.571 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 systemd[1]: Started Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:04:56.571 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.251Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.741984ms db_storage=980ns remote_storage=1.587µs web_handler=386ns query_engine=826ns scrape=425.825µs scrape_sd=97.317µs notify=13.095µs notify_sd=15.185µs rules=21.78956ms tracing=5.917µs 2024-06-18T03:04:56.571 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.251Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-06-18T03:04:56.571 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:04:56.252Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-06-18T03:04:56.922 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-mon[25944]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:56.922 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:56.922 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:56 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:57.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:56 smithi163 ceph-mon[29791]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:57.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:56 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:57.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:56 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:57.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:56 smithi163 ceph-mon[25094]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:57.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:56 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:57.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:56 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:04:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:04:58.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:04:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:04:58.711 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:04:58.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:04:58 smithi163 ceph-mon[29791]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:58.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:04:58 smithi163 ceph-mon[25094]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:04:58 smithi186 ceph-mon[25944]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:04:59.388 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:04:59.388 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (13m) 21s ago 21m 21.8M - 0.23.0 ba2b418f427c 5e109a483ef4 2024-06-18T03:04:59.388 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (13m) 21s ago 21m 53.4M - 8.3.5 dad864ee21e9 a3fd9ade8374 2024-06-18T03:04:59.388 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 unknown 21s ago 20m 53.4M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:04:59.388 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (47s) 21s ago 26m 439M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:04:59.388 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (10m) 21s ago 27m 492M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:04:59.388 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (27m) 21s ago 27m 81.8M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T03:04:59.388 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (26m) 21s ago 26m 57.6M 2048M 17.2.0 e1d6a67b021e a3618f734a9f 2024-06-18T03:04:59.388 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (26m) 21s ago 26m 56.7M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T03:04:59.389 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (32s) 21s ago 22m 8010k - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:04:59.389 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (26s) 21s ago 22m 3468k - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:04:59.389 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (25m) 21s ago 25m 55.9M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T03:04:59.389 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (25m) 21s ago 25m 56.5M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T03:04:59.389 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (25m) 21s ago 25m 51.1M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:04:59.389 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (24m) 21s ago 24m 54.8M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:04:59.389 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (24m) 21s ago 24m 61.4M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:04:59.389 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (23m) 21s ago 23m 52.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:04:59.389 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (23m) 21s ago 23m 51.9M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:04:59.389 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (23m) 21s ago 23m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:04:59.389 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 starting - - - - 2024-06-18T03:04:59.389 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (20m) 21s ago 20m 78.9M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:04:59.389 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (20m) 21s ago 20m 79.6M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:04:59.778 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:04:59.778 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:04:59.778 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: "mds": {}, 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:04:59.779 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:04:59.780 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:05:00.105 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:05:00.105 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:05:00.105 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:05:00.105 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-06-18T03:05:00.105 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [ 2024-06-18T03:05:00.105 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr" 2024-06-18T03:05:00.105 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2024-06-18T03:05:00.105 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "2/2 daemons upgraded", 2024-06-18T03:05:00.106 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading prometheus daemons", 2024-06-18T03:05:00.106 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:05:00.106 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:05:01.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[29791]: from='client.25432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:01.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[29791]: from='client.15534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:01.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[29791]: from='client.25444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:01.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[29791]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:01.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:01.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/624211519' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:01.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:01.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:01.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[25094]: from='client.25432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:01.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[25094]: from='client.15534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:01.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[25094]: from='client.25444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:01.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[25094]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:01.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:01.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/624211519' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:01.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:01.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:00 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:00 smithi186 ceph-mon[25944]: from='client.25432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:00 smithi186 ceph-mon[25944]: from='client.15534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:00 smithi186 ceph-mon[25944]: from='client.25444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:00 smithi186 ceph-mon[25944]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:00 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:00 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/624211519' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:01.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:00 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:01.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:00 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:01.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:01 smithi186 ceph-mon[25944]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:01.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:01 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:01.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:01 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:02.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:01 smithi163 ceph-mon[29791]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:01 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:01 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:02.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:01 smithi163 ceph-mon[25094]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:02.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:01 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:02.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:01 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:02 smithi186 ceph-mon[25944]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:03.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:02 smithi163 ceph-mon[29791]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:03.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:02 smithi163 ceph-mon[25094]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:04.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:05:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:05:03] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-06-18T03:05:04.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:04.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:04.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:04.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:04.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:04.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:04.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:05:04.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:05:04.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.701 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:04.701 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:04.701 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:04.701 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:04.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:04 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:05:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:05:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:05.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:04 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:06.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:05 smithi163 ceph-mon[29791]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 981 B/s rd, 0 op/s 2024-06-18T03:05:06.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:05 smithi163 ceph-mon[29791]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:06.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:05 smithi163 ceph-mon[29791]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:05:06.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:05 smithi163 ceph-mon[25094]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 981 B/s rd, 0 op/s 2024-06-18T03:05:06.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:05 smithi163 ceph-mon[25094]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:06.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:05 smithi163 ceph-mon[25094]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:05:06.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:05 smithi186 ceph-mon[25944]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 981 B/s rd, 0 op/s 2024-06-18T03:05:06.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:05 smithi186 ceph-mon[25944]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:06.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:05 smithi186 ceph-mon[25944]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-18T03:05:06.969 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:06 smithi186 ceph-mon[25944]: Upgrade: Updating alertmanager.a 2024-06-18T03:05:06.970 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:06 smithi186 ceph-mon[25944]: Deploying daemon alertmanager.a on smithi163 2024-06-18T03:05:07.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:06 smithi163 ceph-mon[25094]: Upgrade: Updating alertmanager.a 2024-06-18T03:05:07.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:06 smithi163 ceph-mon[25094]: Deploying daemon alertmanager.a on smithi163 2024-06-18T03:05:07.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:06 smithi163 ceph-mon[29791]: Upgrade: Updating alertmanager.a 2024-06-18T03:05:07.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:06 smithi163 ceph-mon[29791]: Deploying daemon alertmanager.a on smithi163 2024-06-18T03:05:07.985 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:07 smithi186 ceph-mon[25944]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:08.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:07 smithi163 ceph-mon[29791]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:08.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:07 smithi163 ceph-mon[25094]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:10.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:09 smithi163 ceph-mon[29791]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-06-18T03:05:10.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:09 smithi163 ceph-mon[25094]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-06-18T03:05:10.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:09 smithi186 ceph-mon[25944]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-06-18T03:05:11.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:10 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:11.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:10 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:11.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:10 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:12.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:11 smithi163 ceph-mon[29791]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:12.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:11 smithi163 ceph-mon[25094]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:12.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:11 smithi186 ceph-mon[25944]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:13.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:12 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:05:13.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:12 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:05:13.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:12 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:05:13.950 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:13 smithi163 ceph-mon[29791]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-06-18T03:05:13.951 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:05:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:05:13] "GET /metrics HTTP/1.1" 200 37989 "" "Prometheus/2.43.0" 2024-06-18T03:05:13.951 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:13 smithi163 ceph-mon[25094]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-06-18T03:05:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:13 smithi186 ceph-mon[25944]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-06-18T03:05:15.042 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:14 smithi163 systemd[1]: Stopping Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:05:15.434 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[59075]: level=info ts=2024-06-18T03:05:15.126Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-06-18T03:05:15.435 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:15 smithi163 podman[77935]: 2024-06-18 03:05:15.167208487 +0000 UTC m=+0.164292889 container died 5e109a483ef410ecf5902d8a6eeb474d0b41a09f4181c5528b3983242913747f (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T03:05:15.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:15 smithi163 ceph-mon[29791]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:05:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:15 smithi163 ceph-mon[25094]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:05:15.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:15 smithi163 podman[77935]: 2024-06-18 03:05:15.525803126 +0000 UTC m=+0.522887523 container cleanup 5e109a483ef410ecf5902d8a6eeb474d0b41a09f4181c5528b3983242913747f (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T03:05:15.793 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:15 smithi163 bash[77935]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a 2024-06-18T03:05:16.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:15 smithi186 ceph-mon[25944]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:05:16.273 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:15 smithi163 podman[77948]: 2024-06-18 03:05:15.847071127 +0000 UTC m=+0.679455106 container remove 5e109a483ef410ecf5902d8a6eeb474d0b41a09f4181c5528b3983242913747f (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T03:05:16.273 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:15 smithi163 podman[77948]: 2024-06-18 03:05:15.928085443 +0000 UTC m=+0.760469421 volume remove 91054bb2351ff22cb4b6e451b3b4fd0233e60bf1919bdcd624aeda611d619368 2024-06-18T03:05:16.274 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-06-18T03:05:16.274 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 systemd[1]: Stopped Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:05:16.274 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@alertmanager.a.service: Consumed 1.485s CPU time. 2024-06-18T03:05:16.274 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 systemd[1]: Starting Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:05:16.542 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 podman[78096]: 2024-06-18 03:05:16.4793394 +0000 UTC m=+0.107361785 volume create 10a06faa0e0061e630000a281694b371f385a2012070fc753935adabe69b47c4 2024-06-18T03:05:16.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 podman[78096]: 2024-06-18 03:05:16.389423211 +0000 UTC m=+0.017445537 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-06-18T03:05:16.932 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:16 smithi163 ceph-mon[29791]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:16.932 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:16 smithi163 ceph-mon[25094]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:16.932 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 podman[78096]: 2024-06-18 03:05:16.596260296 +0000 UTC m=+0.224282625 container create 18fe92de29b081c82c028060524c3d79dfbb13c8995fedad0d037a0fa2d20206 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T03:05:17.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:16 smithi186 ceph-mon[25944]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:17.292 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 podman[78096]: 2024-06-18 03:05:16.932337131 +0000 UTC m=+0.560359457 container init 18fe92de29b081c82c028060524c3d79dfbb13c8995fedad0d037a0fa2d20206 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T03:05:17.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 podman[78096]: 2024-06-18 03:05:16.935514753 +0000 UTC m=+0.563537083 container start 18fe92de29b081c82c028060524c3d79dfbb13c8995fedad0d037a0fa2d20206 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T03:05:17.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:05:16.948Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-06-18T03:05:17.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:05:16.948Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-06-18T03:05:17.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:05:16.949Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.163 port=9094 2024-06-18T03:05:17.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:05:16.950Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-06-18T03:05:17.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:05:16.973Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-18T03:05:17.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:05:16.973Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-18T03:05:17.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:05:16.975Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-06-18T03:05:17.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:16 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:05:16.975Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-06-18T03:05:17.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:17 smithi163 bash[78096]: 18fe92de29b081c82c028060524c3d79dfbb13c8995fedad0d037a0fa2d20206 2024-06-18T03:05:17.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:17 smithi163 systemd[1]: Started Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:05:18.386 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:18 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:18.387 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:18 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:18.391 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:18 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:18.391 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:18 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:18.391 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:18 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:18.391 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:18 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:19.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:19 smithi163 ceph-mon[29791]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:19.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:19 smithi163 ceph-mon[25094]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:19.293 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:18 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:05:18.950Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000112539s 2024-06-18T03:05:19.414 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:19 smithi186 ceph-mon[25944]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:20.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:20 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:20.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:20 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:20.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:20 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:22.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:21 smithi163 ceph-mon[25094]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:22.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:22.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:22.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:22.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:21 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:22.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:21 smithi163 ceph-mon[29791]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:22.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:22.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:22.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:22.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:21 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:22.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:21 smithi186 ceph-mon[25944]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:22.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:22.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:22.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:22.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:21 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:23.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:22 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:23.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:22 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:23.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:22 smithi163 ceph-mon[29791]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:23.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:22 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:23.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:22 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:23.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:22 smithi163 ceph-mon[25094]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:23.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:22 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:23.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:22 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:23.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:22 smithi186 ceph-mon[25944]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:24.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:05:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:05:23] "GET /metrics HTTP/1.1" 200 37989 "" "Prometheus/2.43.0" 2024-06-18T03:05:25.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:24 smithi163 ceph-mon[25094]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:25.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:24 smithi163 ceph-mon[29791]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:25.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:24 smithi186 ceph-mon[25944]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:27.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:27.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:27.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:27.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:27.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:27.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 0 op/s 2024-06-18T03:05:27.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:27.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:27.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T03:05:27.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T03:05:27.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:05:27.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.195 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:05:26.954Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003821634s 2024-06-18T03:05:27.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:27.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:27.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:27.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:27.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:27.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 0 op/s 2024-06-18T03:05:27.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:27.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:27.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T03:05:27.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T03:05:27.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:05:27.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.196 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:27.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:27.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:27.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:27.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:27.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 0 op/s 2024-06-18T03:05:27.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:27.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:27.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T03:05:27.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-18T03:05:27.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:05:27.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.199 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.199 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.199 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.199 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:27.199 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:28.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:27 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:05:28.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:27 smithi163 ceph-mon[25094]: Upgrade: Updating grafana.a 2024-06-18T03:05:28.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:27 smithi163 ceph-mon[25094]: Deploying daemon grafana.a on smithi186 2024-06-18T03:05:28.205 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:27 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:05:28.205 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:27 smithi163 ceph-mon[29791]: Upgrade: Updating grafana.a 2024-06-18T03:05:28.206 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:27 smithi163 ceph-mon[29791]: Deploying daemon grafana.a on smithi186 2024-06-18T03:05:28.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:27 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:05:28.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:27 smithi186 ceph-mon[25944]: Upgrade: Updating grafana.a 2024-06-18T03:05:28.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:27 smithi186 ceph-mon[25944]: Deploying daemon grafana.a on smithi186 2024-06-18T03:05:29.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:28 smithi163 ceph-mon[29791]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 636 B/s rd, 0 op/s 2024-06-18T03:05:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:28 smithi163 ceph-mon[25094]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 636 B/s rd, 0 op/s 2024-06-18T03:05:29.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:28 smithi186 ceph-mon[25944]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 636 B/s rd, 0 op/s 2024-06-18T03:05:30.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:29 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:30.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:29 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:30.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:29 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:30.462 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:05:31.105 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:05:31.105 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (14s) 9s ago 22m 16.2M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:05:31.105 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (14m) 10s ago 21m 53.1M - 8.3.5 dad864ee21e9 a3fd9ade8374 2024-06-18T03:05:31.105 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 unknown 9s ago 21m 57.1M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:05:31.105 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (78s) 10s ago 26m 440M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:05:31.105 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (11m) 9s ago 28m 497M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:05:31.105 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (28m) 9s ago 28m 85.6M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T03:05:31.105 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (26m) 10s ago 26m 62.0M 2048M 17.2.0 e1d6a67b021e a3618f734a9f 2024-06-18T03:05:31.106 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (27m) 9s ago 27m 61.2M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T03:05:31.106 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (64s) 9s ago 22m 8837k - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:05:31.106 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (58s) 10s ago 22m 7914k - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:05:31.106 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (26m) 9s ago 26m 55.8M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T03:05:31.106 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (26m) 9s ago 26m 56.5M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T03:05:31.106 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (25m) 9s ago 25m 51.1M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:05:31.106 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (25m) 9s ago 25m 54.8M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:05:31.106 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (24m) 10s ago 24m 61.4M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:05:31.106 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (24m) 10s ago 24m 52.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:05:31.106 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (24m) 10s ago 24m 51.9M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:05:31.106 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (23m) 10s ago 23m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:05:31.106 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (35s) 10s ago 22m 53.0M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:05:31.107 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (21m) 9s ago 21m 79.1M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:05:31.107 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (21m) 10s ago 21m 79.8M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:05:31.497 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:05:31.497 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:05:31.497 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-18T03:05:31.497 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:05:31.497 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:05:31.497 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:05:31.497 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:05:31.497 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:05:31.498 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-18T03:05:31.498 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:05:31.498 INFO:teuthology.orchestra.run.smithi163.stdout: "mds": {}, 2024-06-18T03:05:31.498 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:05:31.498 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:05:31.498 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:05:31.498 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:05:31.498 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-18T03:05:31.498 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:05:31.498 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:05:31.498 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:05:31.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:31 smithi163 ceph-mon[25094]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:31.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:31 smithi163 ceph-mon[25094]: from='client.25462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:31.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:31 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/3659491981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:31.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:31 smithi163 ceph-mon[29791]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:31.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:31 smithi163 ceph-mon[29791]: from='client.25462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:31.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:31 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/3659491981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:31.823 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:05:31.823 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:05:31.823 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:05:31.823 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-06-18T03:05:31.823 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [ 2024-06-18T03:05:31.823 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr" 2024-06-18T03:05:31.823 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2024-06-18T03:05:31.824 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "2/2 daemons upgraded", 2024-06-18T03:05:31.824 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading grafana daemons", 2024-06-18T03:05:31.824 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:05:31.824 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:05:32.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:31 smithi186 ceph-mon[25944]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:32.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:31 smithi186 ceph-mon[25944]: from='client.25462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:32.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:31 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/3659491981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:33.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:32 smithi163 ceph-mon[29791]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:33.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:32 smithi163 ceph-mon[29791]: from='client.25474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:33.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:32 smithi163 ceph-mon[25094]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:33.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:32 smithi163 ceph-mon[25094]: from='client.25474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:33.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:32 smithi186 ceph-mon[25944]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:33.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:32 smithi186 ceph-mon[25944]: from='client.25474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:33.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:33 smithi163 ceph-mon[25094]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:33.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:33 smithi163 ceph-mon[25094]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 636 B/s rd, 0 op/s 2024-06-18T03:05:33.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:33 smithi163 ceph-mon[29791]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:33.854 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:33 smithi163 ceph-mon[29791]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 636 B/s rd, 0 op/s 2024-06-18T03:05:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:33 smithi186 ceph-mon[25944]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:05:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:33 smithi186 ceph-mon[25944]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 636 B/s rd, 0 op/s 2024-06-18T03:05:34.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:05:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:05:33] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.43.0" 2024-06-18T03:05:36.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:35 smithi163 ceph-mon[29791]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:36.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:35 smithi163 ceph-mon[25094]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:36.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:35 smithi186 ceph-mon[25944]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:37 smithi186 ceph-mon[25944]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:05:38.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:37 smithi163 ceph-mon[29791]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:05:38.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:37 smithi163 ceph-mon[25094]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:05:40.021 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:39 smithi186 ceph-mon[25944]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:40.021 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:39 smithi186 systemd[1]: Stopping Ceph grafana.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:05:40.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:39 smithi163 ceph-mon[29791]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:40.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:39 smithi163 ceph-mon[25094]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:40.318 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:40 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[49581]: t=2024-06-18T03:05:40+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-06-18T03:05:40.318 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:40 smithi186 podman[66014]: 2024-06-18 03:05:40.031355644 +0000 UTC m=+0.117916448 container died a3fd9ade837492165c6d08920beb7df7fc34e05e2fd9a58e6194129517d04df1 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, io.buildah.version=1.24.2, vcs-type=git, description=Ceph Grafana Container, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., version=8.5, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=236.1648460182, maintainer=Paul Cuzner ) 2024-06-18T03:05:40.602 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:40 smithi186 podman[66014]: 2024-06-18 03:05:40.404070356 +0000 UTC m=+0.490631163 container cleanup a3fd9ade837492165c6d08920beb7df7fc34e05e2fd9a58e6194129517d04df1 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, description=Ceph Grafana Container, io.openshift.expose-services=, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, distribution-scope=public, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, vcs-type=git, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2, io.openshift.tags=base rhel8, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-06-18T03:05:40.602 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:40 smithi186 bash[66014]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a 2024-06-18T03:05:41.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:40 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:41.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:40 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:41.046 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:40 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:41.046 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:40 smithi186 podman[66026]: 2024-06-18 03:05:40.705641547 +0000 UTC m=+0.671856420 container remove a3fd9ade837492165c6d08920beb7df7fc34e05e2fd9a58e6194129517d04df1 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vendor=Red Hat, Inc., io.buildah.version=1.24.2, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, name=ubi8, release=236.1648460182, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-06-18T03:05:41.047 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:40 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@grafana.a.service: Deactivated successfully. 2024-06-18T03:05:41.047 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:40 smithi186 systemd[1]: Stopped Ceph grafana.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:05:41.047 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:40 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@grafana.a.service: Consumed 5.548s CPU time. 2024-06-18T03:05:41.318 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 systemd[1]: Starting Ceph grafana.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:05:41.318 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 podman[66157]: 2024-06-18 03:05:41.162136336 +0000 UTC m=+0.017448733 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-06-18T03:05:41.318 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 podman[66157]: 2024-06-18 03:05:41.273115213 +0000 UTC m=+0.128427634 container create 3fb1675854efe7724486edb1e728588521ac1cdcea6d4325db8732c9d4096d3d (image=quay.io/ceph/grafana:9.4.12, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-06-18T03:05:41.632 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-mon[25944]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:41.632 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 podman[66157]: 2024-06-18 03:05:41.475221504 +0000 UTC m=+0.330533895 container init 3fb1675854efe7724486edb1e728588521ac1cdcea6d4325db8732c9d4096d3d (image=quay.io/ceph/grafana:9.4.12, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-06-18T03:05:41.633 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 podman[66157]: 2024-06-18 03:05:41.479493884 +0000 UTC m=+0.334806265 container start 3fb1675854efe7724486edb1e728588521ac1cdcea6d4325db8732c9d4096d3d (image=quay.io/ceph/grafana:9.4.12, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-06-18T03:05:41.633 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 bash[66157]: 3fb1675854efe7724486edb1e728588521ac1cdcea6d4325db8732c9d4096d3d 2024-06-18T03:05:41.633 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 systemd[1]: Started Ceph grafana.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:05:42.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:41 smithi163 ceph-mon[25094]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:42.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:41 smithi163 ceph-mon[29791]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:42.071 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.63305607Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-06-18T03:05:42.071 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633243346Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-06-18T03:05:42.071 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633299249Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-06-18T03:05:42.071 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633307691Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-06-18T03:05:42.071 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633312805Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-06-18T03:05:42.071 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633318257Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-06-18T03:05:42.071 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633322704Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-06-18T03:05:42.071 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633327307Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-06-18T03:05:42.072 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633332106Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-06-18T03:05:42.072 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633336966Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-06-18T03:05:42.072 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633341784Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-06-18T03:05:42.072 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633346405Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-06-18T03:05:42.072 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633351127Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-06-18T03:05:42.072 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633356425Z level=info msg="Path Home" path=/usr/share/grafana 2024-06-18T03:05:42.072 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633361314Z level=info msg="Path Data" path=/var/lib/grafana 2024-06-18T03:05:42.072 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.63336557Z level=info msg="Path Logs" path=/var/log/grafana 2024-06-18T03:05:42.072 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633370087Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-06-18T03:05:42.072 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.63337645Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-06-18T03:05:42.073 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=settings t=2024-06-18T03:05:41.633381218Z level=info msg="App mode production" 2024-06-18T03:05:42.073 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=sqlstore t=2024-06-18T03:05:41.633434728Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-06-18T03:05:42.073 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=sqlstore t=2024-06-18T03:05:41.633450834Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-06-18T03:05:42.074 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.645012069Z level=info msg="Starting DB migrations" 2024-06-18T03:05:42.074 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.648608779Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-06-18T03:05:42.074 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.660340259Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-06-18T03:05:42.074 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.661667159Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-06-18T03:05:42.074 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.662467666Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-06-18T03:05:42.074 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.663544747Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-06-18T03:05:42.074 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.664651496Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-06-18T03:05:42.074 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.665745308Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-06-18T03:05:42.074 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.666589264Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-06-18T03:05:42.074 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.667469559Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-06-18T03:05:42.074 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.668393208Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-06-18T03:05:42.074 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.669194614Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-06-18T03:05:42.074 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.670286206Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.671126602Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.672204662Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.673266016Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.674349842Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.675206236Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.676275417Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.677361047Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.678271081Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.679382177Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.680452447Z level=info msg="Executing migration" id="create provenance_type table" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.681298833Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.682200824Z level=info msg="Executing migration" id="create alert_image table" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.683028135Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-06-18T03:05:42.075 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.683476837Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.683755852Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.684182588Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.684631574Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.684709121Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.684986998Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.685527336Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.685958832Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.686263444Z level=info msg="Executing migration" id="create secrets table" 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.686703279Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.692915626Z level=info msg="Executing migration" id="add name column into data_keys" 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.694040896Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.69435675Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.70054522Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-06-18T03:05:42.076 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.70683634Z level=info msg="Executing migration" id="add column hidden to role table" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.707994288Z level=info msg="Executing migration" id="create query_history table v1" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.708392654Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.708867774Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.709183758Z level=info msg="Executing migration" id="teams permissions migration" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.709598671Z level=info msg="Executing migration" id="dashboard permissions" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.71091485Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.711606982Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.711912246Z level=info msg="Executing migration" id="alerting notification permissions" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.712281419Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.712662022Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.713117284Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.714251128Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.71457903Z level=info msg="Executing migration" id="create correlation table v1" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.715046665Z level=info msg="Executing migration" id="add index correlations.uid" 2024-06-18T03:05:42.077 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.715519679Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-06-18T03:05:42.078 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.715957713Z level=info msg="Executing migration" id="add correlation config column" 2024-06-18T03:05:42.078 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.717105591Z level=info msg="Executing migration" id="create entity_events table" 2024-06-18T03:05:42.078 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.717498872Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-06-18T03:05:42.078 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.717924686Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-06-18T03:05:42.078 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.718015467Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-06-18T03:05:42.078 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.718294985Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-06-18T03:05:42.078 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.718380164Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-06-18T03:05:42.078 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.718665614Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-06-18T03:05:42.078 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.719116921Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-06-18T03:05:42.078 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.721914482Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-06-18T03:05:42.078 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.722396283Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-06-18T03:05:42.078 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.725044475Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-06-18T03:05:42.078 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.727684643Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.728457313Z level=info msg="Executing migration" id="Drop public config table" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.728909917Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.729708624Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.730527293Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.731341449Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.731809961Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.735663989Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.736826309Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.737986221Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.738307053Z level=info msg="Executing migration" id="add share column" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.739519405Z level=info msg="Executing migration" id="create default alerting folders" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.740294779Z level=info msg="Executing migration" id="create file table" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.741165491Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.741626118Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-06-18T03:05:42.079 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.742458028Z level=info msg="Executing migration" id="create file_meta table" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.743298733Z level=info msg="Executing migration" id="file table idx: path key" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.744162889Z level=info msg="Executing migration" id="set path collation in file table" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.744479059Z level=info msg="Executing migration" id="managed permissions migration" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.745562216Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.746408777Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.747249662Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.748466422Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.749312291Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.750182964Z level=info msg="Executing migration" id="update group index for alert rules" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.751003713Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.751570798Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.752390743Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.753646217Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-06-18T03:05:42.080 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.754869116Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.755714483Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.769002206Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.769839949Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.770722011Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.775419908Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.775957831Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.778660746Z level=info msg="Executing migration" id="create folder table" 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.781427597Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.784264618Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.784735038Z level=info msg="Executing migration" id="Update folder title length" 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.787572081Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=migrator t=2024-06-18T03:05:41.790375841Z level=info msg="migrations completed" performed=105 skipped=376 duration=141.809942ms 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=sqlstore t=2024-06-18T03:05:41.790706475Z level=info msg="Created default organization" 2024-06-18T03:05:42.081 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=plugin.finder t=2024-06-18T03:05:41.824719398Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-06-18T03:05:42.082 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=secrets t=2024-06-18T03:05:41.824836799Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-06-18T03:05:42.082 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=query_data t=2024-06-18T03:05:41.827253853Z level=info msg="Query Service initialization" 2024-06-18T03:05:42.082 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:41 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=live.push_http t=2024-06-18T03:05:41.831329539Z level=info msg="Live Push Gateway initialization" 2024-06-18T03:05:42.513 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=infra.usagestats.collector t=2024-06-18T03:05:42.489816724Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-06-18T03:05:42.513 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=provisioning.datasources t=2024-06-18T03:05:42.491686377Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-06-18T03:05:42.513 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=provisioning.datasources t=2024-06-18T03:05:42.491694727Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-06-18T03:05:42.514 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=provisioning.datasources t=2024-06-18T03:05:42.491789923Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-06-18T03:05:42.514 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=provisioning.alerting t=2024-06-18T03:05:42.513901425Z level=info msg="starting to provision alerting" 2024-06-18T03:05:42.514 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=provisioning.alerting t=2024-06-18T03:05:42.513912886Z level=info msg="finished to provision alerting" 2024-06-18T03:05:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:05:42.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=grafanaStorageLogger t=2024-06-18T03:05:42.514232006Z level=info msg="storage starting" 2024-06-18T03:05:42.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=ngalert.state.manager t=2024-06-18T03:05:42.514470322Z level=info msg="Warming state cache for startup" 2024-06-18T03:05:42.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=ngalert.state.manager t=2024-06-18T03:05:42.514632081Z level=info msg="State cache has been initialized" states=0 duration=161.341µs 2024-06-18T03:05:42.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=ticker t=2024-06-18T03:05:42.51477264Z level=info msg=starting first_tick=2024-06-18T03:05:50Z 2024-06-18T03:05:42.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=ngalert.multiorg.alertmanager t=2024-06-18T03:05:42.514844501Z level=info msg="starting MultiOrg Alertmanager" 2024-06-18T03:05:42.819 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:05:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=http.server t=2024-06-18T03:05:42.517392093Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-06-18T03:05:43.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:05:43.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:43.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:43.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:05:44.042 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:43 smithi186 ceph-mon[25944]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:44.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:43 smithi163 ceph-mon[25094]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:44.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:05:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:05:43] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.43.0" 2024-06-18T03:05:44.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:43 smithi163 ceph-mon[29791]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:46.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:45 smithi163 ceph-mon[29791]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:46.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:45 smithi163 ceph-mon[25094]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:45 smithi186 ceph-mon[25944]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:47.034 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:46 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:47.034 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:46 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:47.034 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:46 smithi186 ceph-mon[25944]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:47.034 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:46 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:47.034 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:46 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:47.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:46 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:47.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:46 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:47.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:46 smithi163 ceph-mon[29791]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:47.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:46 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:47.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:46 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:47.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:46 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:47.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:46 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:47.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:46 smithi163 ceph-mon[25094]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:47.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:46 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:47.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:46 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:49.245 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:49 smithi186 ceph-mon[25944]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:49.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:49 smithi163 ceph-mon[29791]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:49.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:49 smithi163 ceph-mon[25094]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:05:50.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:50.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:50.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:50.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T03:05:50.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:05:50.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:05:50.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:05:50.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.011 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:05:50.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:50.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:50.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:50.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.012 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:05:50.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:05:50.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.013 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:05:50.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:05:50.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:05:50.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:05:50.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:05:50.013 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.015 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.015 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.015 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:05:50.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:05:50.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:05:50.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:05:50.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:05:50.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:05:50.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:05:50.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:50.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:50.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.017 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:05:50.017 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:05:50.017 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:50.017 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:50.017 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:50 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:05:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T03:05:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:05:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:05:50.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:05:50.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:05:50.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:05:50.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:05:50.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:05:50.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:05:50.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:05:50.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:05:50.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:05:50.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:05:50.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:05:50.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:05:50.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:05:50.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:05:50.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:05:50.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:05:50.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:05:50.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:05:50.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:05:50.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:05:50.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:50.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:50.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:50 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[29791]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:05:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[29791]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:05:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[29791]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T03:05:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[29791]: Upgrade: Finalizing container_image settings 2024-06-18T03:05:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[29791]: Upgrade: Complete! 2024-06-18T03:05:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[29791]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:05:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[29791]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 0 op/s 2024-06-18T03:05:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:51.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[29791]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:51.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[29791]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-06-18T03:05:51.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:51.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[25094]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:05:51.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[25094]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:05:51.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[25094]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T03:05:51.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[25094]: Upgrade: Finalizing container_image settings 2024-06-18T03:05:51.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[25094]: Upgrade: Complete! 2024-06-18T03:05:51.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[25094]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:05:51.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[25094]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 0 op/s 2024-06-18T03:05:51.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:51.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:51.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[25094]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:51.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[25094]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-06-18T03:05:51.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:51 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:51.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:51 smithi186 ceph-mon[25944]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:05:51.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:51 smithi186 ceph-mon[25944]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:05:51.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:51 smithi186 ceph-mon[25944]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-18T03:05:51.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:51 smithi186 ceph-mon[25944]: Upgrade: Finalizing container_image settings 2024-06-18T03:05:51.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:51 smithi186 ceph-mon[25944]: Upgrade: Complete! 2024-06-18T03:05:51.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:51 smithi186 ceph-mon[25944]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:05:51.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:51 smithi186 ceph-mon[25944]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 0 op/s 2024-06-18T03:05:51.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:51 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:05:51.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:51 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:05:51.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:51 smithi186 ceph-mon[25944]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:51.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:51 smithi186 ceph-mon[25944]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-06-18T03:05:51.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:51 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:53.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:53 smithi163 ceph-mon[29791]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:05:53.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:53 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:53.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:53 smithi163 ceph-mon[25094]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:05:53.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:53 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:53 smithi186 ceph-mon[25944]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:05:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:53 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:05:54.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:05:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:05:53] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.43.0" 2024-06-18T03:05:55.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:55 smithi163 ceph-mon[25094]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:55.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:55 smithi163 ceph-mon[29791]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:55.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:55 smithi186 ceph-mon[25944]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:05:58.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:57 smithi163 ceph-mon[29791]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-06-18T03:05:58.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:05:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:57 smithi163 ceph-mon[25094]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-06-18T03:05:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:05:58.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:57 smithi186 ceph-mon[25944]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-06-18T03:05:58.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:57 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:06:00.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:05:59 smithi163 ceph-mon[29791]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:00.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:05:59 smithi163 ceph-mon[25094]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:00.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:05:59 smithi186 ceph-mon[25944]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:02.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:01 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:02.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:01 smithi163 ceph-mon[25094]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:06:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:01 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:01 smithi163 ceph-mon[29791]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:06:02.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:01 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:02.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:01 smithi186 ceph-mon[25944]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:06:02.815 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-06-18T03:06:03.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:03 smithi163 ceph-mon[29791]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:03.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:03 smithi163 ceph-mon[29791]: from='client.15588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:03.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:03 smithi163 ceph-mon[25094]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:03.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:03 smithi163 ceph-mon[25094]: from='client.15588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:04.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:03 smithi186 ceph-mon[25944]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:04.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:03 smithi186 ceph-mon[25944]: from='client.15588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:04.156 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:06:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:06:03] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-06-18T03:06:04.931 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:06:05.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-06-18T03:06:05.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:05 smithi163 ceph-mon[25094]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:05.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:05 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/437101597' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:05.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:05 smithi163 ceph-mon[29791]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:05.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:05 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/437101597' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:06.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:05 smithi186 ceph-mon[25944]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:06.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:05 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/437101597' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:07.591 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)" 2024-06-18T03:06:07.871 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:07 smithi186 ceph-mon[25944]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:07.871 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:07 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/850373438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:07.943 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:07 smithi163 ceph-mon[29791]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:07.943 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:07 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/850373438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:07.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:07 smithi163 ceph-mon[25094]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:07.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:07 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/850373438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:08.223 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-06-18T03:06:09.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:09 smithi163 ceph-mon[29791]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:09.730 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:09 smithi163 ceph-mon[25094]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:10.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:09 smithi186 ceph-mon[25944]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:10.234 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:06:10.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-06-18T03:06:11.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:10 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1640453793' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:11.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:10 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1640453793' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:11.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:10 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/1640453793' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:11.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:11 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:11.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:11 smithi163 ceph-mon[25094]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:11.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:11 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:11.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:11 smithi163 ceph-mon[29791]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:12.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:11 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:12.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:11 smithi186 ceph-mon[25944]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:13.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:12 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:06:13.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:12 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:06:13.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:12 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:06:13.991 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:13 smithi163 ceph-mon[25094]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:13.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:13 smithi163 ceph-mon[29791]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:13.992 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:06:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:06:13] "GET /metrics HTTP/1.1" 200 38003 "" "Prometheus/2.43.0" 2024-06-18T03:06:14.033 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:06:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:13 smithi186 ceph-mon[25944]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:14.610 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade status' 2024-06-18T03:06:14.914 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:14 smithi163 ceph-mon[29791]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:14.915 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:14 smithi163 ceph-mon[25094]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:14 smithi186 ceph-mon[25944]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:15.877 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:15 smithi163 ceph-mon[29791]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:15.877 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:15 smithi163 ceph-mon[25094]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:16.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:15 smithi186 ceph-mon[25944]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:16.621 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:06:16.621 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": null, 2024-06-18T03:06:16.621 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": false, 2024-06-18T03:06:16.621 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "", 2024-06-18T03:06:16.621 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:06:16.621 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": null, 2024-06-18T03:06:16.621 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "", 2024-06-18T03:06:16.621 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:06:16.621 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:06:17.148 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph health detail' 2024-06-18T03:06:17.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:17 smithi163 ceph-mon[29791]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:17.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:17 smithi163 ceph-mon[29791]: from='client.25522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:17.729 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:17 smithi163 ceph-mon[25094]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:17.729 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:17 smithi163 ceph-mon[25094]: from='client.25522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:18.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:17 smithi186 ceph-mon[25944]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:18.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:17 smithi186 ceph-mon[25944]: from='client.25522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:19.310 INFO:teuthology.orchestra.run.smithi163.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:06:19.311 INFO:teuthology.orchestra.run.smithi163.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:06:19.311 INFO:teuthology.orchestra.run.smithi163.stdout: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in unknown state 2024-06-18T03:06:19.889 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:19 smithi163 ceph-mon[29791]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:19.889 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:19 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/176163145' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:06:19.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:19 smithi163 ceph-mon[25094]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:19.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:19 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/176163145' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:06:19.912 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-06-18T03:06:20.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:19 smithi186 ceph-mon[25944]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:20.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:19 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/176163145' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:06:22.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:21 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:22.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:21 smithi163 ceph-mon[25094]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:22.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:21 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:22.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:21 smithi163 ceph-mon[29791]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:22.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:21 smithi186 ceph-mon[25944]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:22.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:21 smithi186 ceph-mon[25944]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:23.463 INFO:teuthology.orchestra.run.smithi163.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:06:23.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:23 smithi163 ceph-mon[29791]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:23.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:23 smithi163 ceph-mon[29791]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:23.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:23 smithi163 ceph-mon[29791]: from='client.25540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "mon", "hosts": "smithi186", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:23.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:23 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:23 smithi163 ceph-mon[25094]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:23 smithi163 ceph-mon[25094]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:23 smithi163 ceph-mon[25094]: from='client.25540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "mon", "hosts": "smithi186", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:23 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:24.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:23 smithi186 ceph-mon[25944]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:24.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:23 smithi186 ceph-mon[25944]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:24.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:23 smithi186 ceph-mon[25944]: from='client.25540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "mon", "hosts": "smithi186", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:24.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:23 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:24.168 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:06:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:06:23] "GET /metrics HTTP/1.1" 200 38003 "" "Prometheus/2.43.0" 2024-06-18T03:06:24.195 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-18T03:06:24.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[29791]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:06:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:06:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[29791]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:06:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[29791]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:06:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[29791]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:06:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[25094]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:06:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:06:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[25094]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:06:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[25094]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:06:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:24.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:24 smithi163 ceph-mon[25094]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:06:25.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:24 smithi186 ceph-mon[25944]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:06:25.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:24 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:25.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:24 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:06:25.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:24 smithi186 ceph-mon[25944]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:06:25.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:24 smithi186 ceph-mon[25944]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:06:25.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:24 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:25.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:24 smithi186 ceph-mon[25944]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:06:26.984 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:06:27.280 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:26 smithi186 ceph-mon[25944]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:06:27.280 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:27.280 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:26 smithi186 ceph-mon[25944]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:06:27.280 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:26 smithi186 ceph-mon[25944]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:06:27.280 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:06:27.280 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:27.280 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:26 smithi186 ceph-mon[25944]: Upgrade: Setting container_image for all mgr 2024-06-18T03:06:27.281 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:27.281 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-18T03:06:27.281 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:26 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-06-18T03:06:27.281 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:26 smithi186 ceph-mon[25944]: Upgrade: It appears safe to stop mon.b 2024-06-18T03:06:27.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[29791]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:06:27.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:27.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[29791]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:06:27.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[29791]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:06:27.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:06:27.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:27.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all mgr 2024-06-18T03:06:27.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:27.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-18T03:06:27.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-06-18T03:06:27.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[29791]: Upgrade: It appears safe to stop mon.b 2024-06-18T03:06:27.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[25094]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:06:27.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:27.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[25094]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:06:27.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[25094]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:06:27.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:06:27.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:27.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all mgr 2024-06-18T03:06:27.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:27.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-18T03:06:27.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-06-18T03:06:27.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:26 smithi163 ceph-mon[25094]: Upgrade: It appears safe to stop mon.b 2024-06-18T03:06:27.704 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:06:27.704 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (71s) 66s ago 23m 16.2M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:06:27.705 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (46s) 41s ago 22m 39.6M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:06:27.705 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 unknown 66s ago 22m 57.1M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:06:27.705 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (2m) 41s ago 27m 440M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:06:27.705 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (11m) 66s ago 29m 497M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:06:27.705 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (29m) 66s ago 29m 85.6M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T03:06:27.705 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (27m) 41s ago 27m 53.9M 2048M 17.2.0 e1d6a67b021e a3618f734a9f 2024-06-18T03:06:27.705 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (28m) 66s ago 28m 61.2M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T03:06:27.705 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (2m) 66s ago 23m 8837k - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:06:27.705 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (114s) 41s ago 23m 13.4M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:06:27.705 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (27m) 66s ago 27m 55.8M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T03:06:27.706 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (26m) 66s ago 26m 56.5M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T03:06:27.706 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (26m) 66s ago 26m 51.1M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:06:27.706 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (26m) 66s ago 26m 54.8M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:06:27.706 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (25m) 41s ago 25m 61.4M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:06:27.706 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (25m) 41s ago 25m 52.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:06:27.706 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (25m) 41s ago 25m 51.9M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:06:27.706 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (24m) 41s ago 24m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:06:27.706 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (92s) 41s ago 23m 56.3M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:06:27.706 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (22m) 66s ago 22m 79.1M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:06:27.706 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (22m) 41s ago 22m 79.9M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:06:27.858 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[25094]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:27.942 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:27 smithi186 ceph-mon[25944]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:27.942 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:27 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:06:27.942 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:27 smithi186 ceph-mon[25944]: from='client.25552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:27.942 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:27 smithi186 ceph-mon[25944]: Upgrade: Updating mon.b 2024-06-18T03:06:27.942 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:27 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:27.942 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:27 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:06:27.942 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:27 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:06:27.942 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:27 smithi186 ceph-mon[25944]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:27.942 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:27 smithi186 ceph-mon[25944]: Deploying daemon mon.b on smithi186 2024-06-18T03:06:28.133 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[29791]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:28.134 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:06:28.134 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[29791]: from='client.25552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:28.134 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[29791]: Upgrade: Updating mon.b 2024-06-18T03:06:28.134 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:28.135 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:06:28.135 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:06:28.135 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:28.135 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[29791]: Deploying daemon mon.b on smithi186 2024-06-18T03:06:28.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:06:28.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[25094]: from='client.25552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:28.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[25094]: Upgrade: Updating mon.b 2024-06-18T03:06:28.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:28.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:06:28.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:06:28.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:28.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:27 smithi163 ceph-mon[25094]: Deploying daemon mon.b on smithi186 2024-06-18T03:06:28.137 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:06:28.137 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:06:28.137 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-18T03:06:28.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:06:28.139 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:06:28.139 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:06:28.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:06:28.139 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:06:28.139 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-18T03:06:28.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:06:28.139 INFO:teuthology.orchestra.run.smithi163.stdout: "mds": {}, 2024-06-18T03:06:28.140 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:06:28.140 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:06:28.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:06:28.140 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:06:28.140 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-18T03:06:28.140 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:06:28.140 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:06:28.140 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:06:28.466 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:06:28.467 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:06:28.467 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:06:28.467 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi186", 2024-06-18T03:06:28.467 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:06:28.467 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "0/1 daemons upgraded", 2024-06-18T03:06:28.467 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading mon daemons", 2024-06-18T03:06:28.467 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:06:28.467 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:06:28.880 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:28 smithi186 ceph-mon[25944]: from='client.25558 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:28.881 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:28 smithi186 ceph-mon[25944]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 521 B/s rd, 0 op/s 2024-06-18T03:06:28.881 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:28 smithi186 ceph-mon[25944]: from='client.? 172.21.15.163:0/4147154328' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:28.881 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:28 smithi186 ceph-mon[25944]: from='client.25570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:29.151 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:28 smithi186 systemd[1]: Stopping Ceph mon.b for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:06:29.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:28 smithi163 ceph-mon[25094]: from='client.25558 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:28 smithi163 ceph-mon[25094]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 521 B/s rd, 0 op/s 2024-06-18T03:06:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:28 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/4147154328' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:29.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:28 smithi163 ceph-mon[25094]: from='client.25570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:29.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:28 smithi163 ceph-mon[29791]: from='client.25558 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:29.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:28 smithi163 ceph-mon[29791]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 521 B/s rd, 0 op/s 2024-06-18T03:06:29.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:28 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/4147154328' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:29.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:28 smithi163 ceph-mon[29791]: from='client.25570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:06:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:29 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b[25927]: 2024-06-18T03:06:29.150+0000 7f3bbd844700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:06:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:29 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b[25927]: 2024-06-18T03:06:29.150+0000 7f3bbd844700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-06-18T03:06:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:29 smithi186 podman[67592]: 2024-06-18 03:06:29.385180813 +0000 UTC m=+0.358553434 container died a3618f734a9fdfe387382742815495207284cce3a3e27ae27b23c8bcf40909f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.openshift.expose-services=, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, architecture=x86_64, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD) 2024-06-18T03:06:30.017 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:29 smithi186 podman[67592]: 2024-06-18 03:06:29.758153592 +0000 UTC m=+0.731526211 container cleanup a3618f734a9fdfe387382742815495207284cce3a3e27ae27b23c8bcf40909f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_CLEAN=True, version=8, ceph=True, io.openshift.tags=base centos centos-stream, release=754, GIT_BRANCH=HEAD, architecture=x86_64, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vcs-type=git, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-06-18T03:06:30.018 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:29 smithi186 bash[67592]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b 2024-06-18T03:06:30.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 podman[67605]: 2024-06-18 03:06:30.017923901 +0000 UTC m=+0.632074001 container remove a3618f734a9fdfe387382742815495207284cce3a3e27ae27b23c8bcf40909f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, release=754, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, version=8, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8) 2024-06-18T03:06:30.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.b.service: Deactivated successfully. 2024-06-18T03:06:30.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 systemd[1]: Stopped Ceph mon.b for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:06:30.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.b.service: Consumed 16.934s CPU time. 2024-06-18T03:06:30.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 systemd[1]: Starting Ceph mon.b for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:06:30.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 podman[67733]: 2024-06-18 03:06:30.493290114 +0000 UTC m=+0.017616738 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:06:30.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 podman[67733]: 2024-06-18 03:06:30.6025675 +0000 UTC m=+0.126894166 container create a0f80892befcff66556062ba6dc843e01acd9d2bba7557f65833e262968f9f1e (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_CLEAN=True, RELEASE=squid-3781e3a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:06:30.990 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 podman[67733]: 2024-06-18 03:06:30.888609077 +0000 UTC m=+0.412935710 container init a0f80892befcff66556062ba6dc843e01acd9d2bba7557f65833e262968f9f1e (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-06-18T03:06:30.990 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 podman[67733]: 2024-06-18 03:06:30.891957073 +0000 UTC m=+0.416283706 container start a0f80892befcff66556062ba6dc843e01acd9d2bba7557f65833e262968f9f1e (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:06:30.990 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: set uid:gid to 167:167 (ceph:ceph) 2024-06-18T03:06:30.990 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc), process ceph-mon, pid 2 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: pidfile_write: ignore empty --pid-file 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: load: jerasure load: lrc 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: RocksDB version: 7.9.2 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Git sha 0 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Compile date 2024-06-17 15:21:02 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: DB SUMMARY 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: DB Session ID: OKVB2D344MIF6EB2M1C8 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: CURRENT file: CURRENT 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: IDENTITY file: IDENTITY 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2814 Bytes 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 2132979 ; 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.error_if_exists: 0 2024-06-18T03:06:30.991 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.create_if_missing: 0 2024-06-18T03:06:30.992 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.paranoid_checks: 1 2024-06-18T03:06:30.992 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.flush_verify_memtable_count: 1 2024-06-18T03:06:30.992 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-18T03:06:30.992 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-06-18T03:06:30.992 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.env: 0x55a57b23e740 2024-06-18T03:06:30.992 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.fs: PosixFileSystem 2024-06-18T03:06:30.992 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.info_log: 0x55a57c043000 2024-06-18T03:06:30.992 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_file_opening_threads: 16 2024-06-18T03:06:30.992 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.statistics: (nil) 2024-06-18T03:06:30.993 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.use_fsync: 0 2024-06-18T03:06:30.993 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_log_file_size: 0 2024-06-18T03:06:30.993 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-18T03:06:30.993 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-18T03:06:30.993 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.keep_log_file_num: 1000 2024-06-18T03:06:30.993 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.recycle_log_file_num: 0 2024-06-18T03:06:30.993 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.allow_fallocate: 1 2024-06-18T03:06:30.993 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.allow_mmap_reads: 0 2024-06-18T03:06:30.993 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.allow_mmap_writes: 0 2024-06-18T03:06:30.993 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.use_direct_reads: 0 2024-06-18T03:06:30.993 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-18T03:06:30.994 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.create_missing_column_families: 0 2024-06-18T03:06:30.994 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.db_log_dir: 2024-06-18T03:06:30.994 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.wal_dir: 2024-06-18T03:06:30.994 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-18T03:06:30.994 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-18T03:06:30.994 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-18T03:06:30.994 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-18T03:06:30.994 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-18T03:06:30.994 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-18T03:06:30.994 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.advise_random_on_open: 1 2024-06-18T03:06:30.994 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.db_write_buffer_size: 0 2024-06-18T03:06:30.994 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.write_buffer_manager: 0x55a57c052c80 2024-06-18T03:06:30.994 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.rate_limiter: (nil) 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.wal_recovery_mode: 2 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.enable_thread_tracking: 0 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.enable_pipelined_write: 0 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.unordered_write: 0 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.row_cache: None 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.wal_filter: None 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.allow_ingest_behind: 0 2024-06-18T03:06:30.995 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.two_write_queues: 0 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.manual_wal_flush: 0 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.wal_compression: 0 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.atomic_flush: 0 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.log_readahead_size: 0 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.best_efforts_recovery: 0 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.allow_data_in_errors: 0 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.db_host_id: __hostname__ 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.enforce_single_del_contracts: true 2024-06-18T03:06:30.996 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_background_jobs: 2 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_background_compactions: -1 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_subcompactions: 1 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_total_wal_size: 0 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_open_files: -1 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.bytes_per_sync: 0 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-18T03:06:30.997 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compaction_readahead_size: 0 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_background_flushes: -1 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Compression algorithms supported: 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: kZSTD supported: 0 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: kXpressCompression supported: 0 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: kBZip2Compression supported: 0 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: kLZ4Compression supported: 1 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: kZlibCompression supported: 1 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: kLZ4HCCompression supported: 1 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: kSnappyCompression supported: 1 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: DMutex implementation: pthread_mutex_t 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-06-18T03:06:30.998 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.merge_operator: 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compaction_filter: None 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compaction_filter_factory: None 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.sst_partitioner_factory: None 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a57c042d00) 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout: cache_index_and_filter_blocks: 1 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout: pin_top_level_index_and_filter: 1 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout: index_type: 0 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout: data_block_index_type: 0 2024-06-18T03:06:30.999 INFO:journalctl@ceph.mon.b.smithi186.stdout: index_shortening: 1 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: checksum: 4 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: no_block_cache: 0 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_cache: 0x55a57c039090 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_cache_name: BinnedLRUCache 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_cache_options: 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: capacity : 536870912 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: num_shard_bits : 4 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: strict_capacity_limit : 0 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: high_pri_pool_ratio: 0.000 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_cache_compressed: (nil) 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: persistent_cache: (nil) 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_size: 4096 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_size_deviation: 10 2024-06-18T03:06:31.000 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_restart_interval: 16 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: index_block_restart_interval: 1 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: metadata_block_size: 4096 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: partition_filters: 0 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: use_delta_encoding: 1 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: filter_policy: bloomfilter 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: whole_key_filtering: 1 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: verify_compression: 0 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: read_amp_bytes_per_bit: 0 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: format_version: 5 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: enable_index_compression: 1 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: block_align: 0 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: max_auto_readahead_size: 262144 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: prepopulate_block_cache: 0 2024-06-18T03:06:31.001 INFO:journalctl@ceph.mon.b.smithi186.stdout: initial_auto_readahead_size: 8192 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout: num_file_reads_for_auto_readahead: 2 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.write_buffer_size: 33554432 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_write_buffer_number: 2 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compression: NoCompression 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.bottommost_compression: Disabled 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.prefix_extractor: nullptr 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.num_levels: 7 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-18T03:06:31.002 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compression_opts.level: 32767 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compression_opts.strategy: 0 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compression_opts.enabled: false 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-18T03:06:31.003 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-18T03:06:31.004 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-18T03:06:31.004 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.target_file_size_base: 67108864 2024-06-18T03:06:31.004 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-18T03:06:31.004 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-18T03:06:31.004 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-18T03:06:31.004 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-18T03:06:31.004 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-18T03:06:31.004 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-18T03:06:31.004 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-18T03:06:31.005 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-18T03:06:31.005 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-18T03:06:31.005 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-18T03:06:31.005 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-18T03:06:31.005 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-18T03:06:31.005 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-18T03:06:31.005 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-06-18T03:06:31.006 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.arena_block_size: 1048576 2024-06-18T03:06:31.006 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-18T03:06:31.006 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-18T03:06:31.006 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.disable_auto_compactions: 0 2024-06-18T03:06:31.006 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-18T03:06:31.006 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-18T03:06:31.006 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-18T03:06:31.006 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-18T03:06:31.006 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-18T03:06:31.006 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-18T03:06:31.006 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-18T03:06:31.006 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.table_properties_collectors: 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.inplace_update_support: 0 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.bloom_locality: 0 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.max_successive_merges: 0 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.paranoid_file_checks: 0 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.force_consistency_checks: 1 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.report_bg_io_stats: 0 2024-06-18T03:06:31.007 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.ttl: 2592000 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.enable_blob_files: false 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.min_blob_size: 0 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.blob_file_size: 268435456 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.blob_file_starting_level: 0 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-06-18T03:06:31.008 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-06-18T03:06:31.009 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-18T03:06:31.009 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28761, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-18T03:06:31.009 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-06-18T03:06:31.009 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 30036245-31ce-48f4-9d08-d025b8ce99df 2024-06-18T03:06:31.009 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718679990919007, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-06-18T03:06:31.009 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-06-18T03:06:31.009 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718679990924912, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 1309842, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28758, "largest_seqno": 30047, "table_properties": {"data_size": 1304659, "index_size": 2888, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1285, "raw_key_size": 12259, "raw_average_key_size": 24, "raw_value_size": 1294139, "raw_average_value_size": 2577, "num_data_blocks": 130, "num_entries": 502, "num_filter_entries": 502, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1718679990, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "30036245-31ce-48f4-9d08-d025b8ce99df", "db_session_id": "OKVB2D344MIF6EB2M1C8", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-06-18T03:06:31.009 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718679990925025, "job": 1, "event": "recovery_finished"} 2024-06-18T03:06:31.009 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-06-18T03:06:31.009 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-18T03:06:31.009 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-18T03:06:31.009 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a57c066e00 2024-06-18T03:06:31.009 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: DB pointer 0x55a57c15c000 2024-06-18T03:06:31.010 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-06-18T03:06:31.010 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-06-18T03:06:31.010 INFO:journalctl@ceph.mon.b.smithi186.stdout: ** DB Stats ** 2024-06-18T03:06:31.010 INFO:journalctl@ceph.mon.b.smithi186.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-18T03:06:31.010 INFO:journalctl@ceph.mon.b.smithi186.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-18T03:06:31.010 INFO:journalctl@ceph.mon.b.smithi186.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-18T03:06:31.010 INFO:journalctl@ceph.mon.b.smithi186.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-18T03:06:31.010 INFO:journalctl@ceph.mon.b.smithi186.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-18T03:06:31.010 INFO:journalctl@ceph.mon.b.smithi186.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-18T03:06:31.010 INFO:journalctl@ceph.mon.b.smithi186.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-18T03:06:31.010 INFO:journalctl@ceph.mon.b.smithi186.stdout: 2024-06-18T03:06:31.010 INFO:journalctl@ceph.mon.b.smithi186.stdout: ** Compaction Stats [default] ** 2024-06-18T03:06:31.010 INFO:journalctl@ceph.mon.b.smithi186.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: L0 1/0 1.25 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 464.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: L6 1/0 9.11 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: Sum 2/0 10.36 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 464.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 464.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: ** Compaction Stats [default] ** 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 464.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: 2024-06-18T03:06:31.011 INFO:journalctl@ceph.mon.b.smithi186.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout: Cumulative compaction: 0.00 GB write, 121.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout: Interval compaction: 0.00 GB write, 121.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout: Block cache BinnedLRUCache@0x55a57c039090#2 capacity: 512.00 MB usage: 4.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,3.08 KB,0.000587106%) Misc(1,0.00 KB,0%) 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout: 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: starting mon.b rank 2 at public addrs [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] at bind addrs [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: mon.b@-1(???) e3 preinit fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T03:06:31.012 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: mon.b@-1(???).mds e1 new map 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: mon.b@-1(???).mds e1 print_map 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout: e1 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout: legacy client fscid: -1 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout: 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout: No filesystems configured 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: mon.b@-1(???).osd e111 crush map has features 3314933000854323200, adjusting msgr requires 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: mon.b@-1(???).osd e111 crush map has features 432629239337189376, adjusting msgr requires 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: mon.b@-1(???).osd e111 crush map has features 432629239337189376, adjusting msgr requires 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: mon.b@-1(???).osd e111 crush map has features 432629239337189376, adjusting msgr requires 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 ceph-mon[67751]: mon.b@-1(???).paxosservice(auth 1..25) refresh upgraded, format 0 -> 3 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 bash[67733]: a0f80892befcff66556062ba6dc843e01acd9d2bba7557f65833e262968f9f1e 2024-06-18T03:06:31.013 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:30 smithi186 systemd[1]: Started Ceph mon.b for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:06:32.436 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:32 smithi186 ceph-mon[67751]: mon.b calling monitor election 2024-06-18T03:06:32.437 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:32 smithi186 ceph-mon[67751]: mon.c calling monitor election 2024-06-18T03:06:32.437 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:32 smithi186 ceph-mon[67751]: mon.a calling monitor election 2024-06-18T03:06:32.437 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:32 smithi186 ceph-mon[67751]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-18T03:06:32.437 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:32 smithi186 ceph-mon[67751]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} 2024-06-18T03:06:32.437 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:32 smithi186 ceph-mon[67751]: fsmap 2024-06-18T03:06:32.437 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:32 smithi186 ceph-mon[67751]: osdmap e111: 8 total, 8 up, 8 in 2024-06-18T03:06:32.437 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:32 smithi186 ceph-mon[67751]: mgrmap e41: y(active, since 2m), standbys: x 2024-06-18T03:06:32.437 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:32 smithi186 ceph-mon[67751]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:06:32.437 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:32 smithi186 ceph-mon[67751]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:06:32.437 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:32 smithi186 ceph-mon[67751]: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in unknown state 2024-06-18T03:06:32.437 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:32 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:32.437 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:32 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:32.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[25094]: mon.b calling monitor election 2024-06-18T03:06:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[25094]: mon.c calling monitor election 2024-06-18T03:06:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[25094]: mon.a calling monitor election 2024-06-18T03:06:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[25094]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-18T03:06:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[25094]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} 2024-06-18T03:06:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[25094]: fsmap 2024-06-18T03:06:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[25094]: osdmap e111: 8 total, 8 up, 8 in 2024-06-18T03:06:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[25094]: mgrmap e41: y(active, since 2m), standbys: x 2024-06-18T03:06:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[25094]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:06:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[25094]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:06:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[25094]: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in unknown state 2024-06-18T03:06:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:32.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[29791]: mon.b calling monitor election 2024-06-18T03:06:32.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[29791]: mon.c calling monitor election 2024-06-18T03:06:32.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[29791]: mon.a calling monitor election 2024-06-18T03:06:32.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[29791]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-18T03:06:32.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[29791]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} 2024-06-18T03:06:32.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[29791]: fsmap 2024-06-18T03:06:32.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[29791]: osdmap e111: 8 total, 8 up, 8 in 2024-06-18T03:06:32.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[29791]: mgrmap e41: y(active, since 2m), standbys: x 2024-06-18T03:06:32.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[29791]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:06:32.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[29791]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:06:32.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[29791]: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in unknown state 2024-06-18T03:06:32.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:32.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:32 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:33.435 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:33 smithi186 ceph-mon[67751]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:06:33.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:33 smithi163 ceph-mon[29791]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:06:33.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:33 smithi163 ceph-mon[25094]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:06:34.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:06:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:06:33] "GET /metrics HTTP/1.1" 200 38003 "" "Prometheus/2.43.0" 2024-06-18T03:06:35.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:35 smithi163 ceph-mon[25094]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:06:35.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:35 smithi163 ceph-mon[29791]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:06:35.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:35 smithi186 ceph-mon[67751]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:06:36.250 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:06:35 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:06:35.936+0000 7fb46c5b5640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-06-18T03:06:36.509 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:36 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:36.509 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:36 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:36.509 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:36 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:36.510 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:36 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:36.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:36 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:36.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:36 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:36.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:36 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:36.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:36 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:36.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:36 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:36.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:36 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:36.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:36 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:36.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:36 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:37.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:37 smithi163 ceph-mon[25094]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:37.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:37 smithi163 ceph-mon[29791]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:37.565 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:37 smithi186 ceph-mon[67751]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:39.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:39.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:39.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:06:39.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:06:39.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:06:39.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:39.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:06:39.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:06:39.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:06:39.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:06:39.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:06:39.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:06:39.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:06:39.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:39.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:39.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:06:39.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:06:39.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-06-18T03:06:39.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:06:39.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:06:39.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:39.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:06:39.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:06:39.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:06:39.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:06:39.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:06:39.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:06:39.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:39 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:06:39.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:39.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:06:39.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:39 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:06:40.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Detected new or changed devices on smithi186 2024-06-18T03:06:40.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:06:40.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:06:40.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all crash 2024-06-18T03:06:40.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all mds 2024-06-18T03:06:40.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:06:40.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all nfs 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all node-exporter 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all prometheus 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all alertmanager 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all grafana 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all loki 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all promtail 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Finalizing container_image settings 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:06:40.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:06:40.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:06:40.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:06:40.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:06:40.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:06:40.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:06:40.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Detected new or changed devices on smithi186 2024-06-18T03:06:40.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:06:40.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all crash 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all mds 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all nfs 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all node-exporter 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all prometheus 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all alertmanager 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all grafana 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all loki 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all promtail 2024-06-18T03:06:40.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Finalizing container_image settings 2024-06-18T03:06:40.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:06:40.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:06:40.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:06:40.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:06:40.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:06:40.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:06:40.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:06:40.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:06:40.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:06:40.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: Upgrade: Complete! 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:40.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:06:40.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:06:40.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-06-18T03:06:40.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:40.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: Upgrade: Complete! 2024-06-18T03:06:40.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:06:40.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:06:40.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:40.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:06:40.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:06:40.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-06-18T03:06:40.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:40.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:40.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:06:40.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:06:40.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-06-18T03:06:40.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:40 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:40.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Detected new or changed devices on smithi186 2024-06-18T03:06:40.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:06:40.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:06:40.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all crash 2024-06-18T03:06:40.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mds 2024-06-18T03:06:40.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:06:40.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-18T03:06:40.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:06:40.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all nfs 2024-06-18T03:06:40.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:06:40.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all node-exporter 2024-06-18T03:06:40.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all prometheus 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all alertmanager 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all grafana 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all loki 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all promtail 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Finalizing container_image settings 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:06:40.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: Upgrade: Complete! 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:40.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:06:40.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:06:40.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:06:40.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-06-18T03:06:40.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:40 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:42.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:41 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:42.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:41 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:42.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:41 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:43.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:42 smithi163 ceph-mon[29791]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:06:43.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:06:43.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:42 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:42 smithi163 ceph-mon[25094]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:06:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:06:43.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:42 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:43.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:42 smithi186 ceph-mon[67751]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:06:43.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:42 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:06:43.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:42 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:06:44.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:06:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:06:43] "GET /metrics HTTP/1.1" 200 38011 "" "Prometheus/2.43.0" 2024-06-18T03:06:45.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:44 smithi163 ceph-mon[29791]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:45.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:44 smithi163 ceph-mon[25094]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:45.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:44 smithi186 ceph-mon[67751]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:47.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:46 smithi163 ceph-mon[29791]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-06-18T03:06:47.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:46 smithi163 ceph-mon[25094]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-06-18T03:06:47.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:46 smithi186 ceph-mon[67751]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-06-18T03:06:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:48 smithi186 ceph-mon[67751]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 631 B/s rd, 0 op/s 2024-06-18T03:06:49.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:48 smithi163 ceph-mon[29791]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 631 B/s rd, 0 op/s 2024-06-18T03:06:49.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:48 smithi163 ceph-mon[25094]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 631 B/s rd, 0 op/s 2024-06-18T03:06:51.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:50 smithi163 ceph-mon[29791]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:06:51.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:50 smithi163 ceph-mon[25094]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:06:51.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:50 smithi186 ceph-mon[67751]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:06:52.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:51 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:52.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:51 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:52.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:51 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:06:53.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:52 smithi163 ceph-mon[29791]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:53.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:52 smithi163 ceph-mon[25094]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:53.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:52 smithi186 ceph-mon[67751]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:54.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:06:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:06:53] "GET /metrics HTTP/1.1" 200 38011 "" "Prometheus/2.43.0" 2024-06-18T03:06:55.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:54 smithi163 ceph-mon[29791]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:55.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:54 smithi163 ceph-mon[25094]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:55.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:54 smithi186 ceph-mon[67751]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:06:57.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:56 smithi163 ceph-mon[29791]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:57.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:56 smithi163 ceph-mon[25094]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:57.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:56 smithi186 ceph-mon[67751]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:57.944 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:57 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:06:57.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:57 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:06:58.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:57 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:06:59.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:06:58 smithi163 ceph-mon[29791]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:59.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:06:58 smithi163 ceph-mon[25094]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:59.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:06:58 smithi186 ceph-mon[67751]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:06:59.537 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps' 2024-06-18T03:07:00.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:00 smithi163 ceph-mon[25094]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:00.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:00 smithi163 ceph-mon[25094]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:00.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:00 smithi163 ceph-mon[29791]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:00.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:00 smithi163 ceph-mon[29791]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:00 smithi186 ceph-mon[67751]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:01.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:00 smithi186 ceph-mon[67751]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:01.551 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (105s) 100s ago 23m 16.2M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (80s) 26s ago 23m 76.5M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 unknown 100s ago 22m 57.1M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (2m) 26s ago 28m 441M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (12m) 100s ago 29m 497M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (29m) 100s ago 30m 85.6M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (31s) 26s ago 28m 21.5M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (28m) 100s ago 28m 61.2M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (2m) 100s ago 24m 8837k - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (2m) 26s ago 24m 15.8M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (27m) 100s ago 27m 55.8M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (27m) 100s ago 27m 56.5M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (27m) 100s ago 27m 51.1M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:07:01.552 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (26m) 100s ago 26m 54.8M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:07:01.553 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (26m) 26s ago 26m 61.5M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:07:01.553 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (25m) 26s ago 25m 52.4M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:07:01.553 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (25m) 26s ago 25m 52.2M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:07:01.553 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (25m) 26s ago 25m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:07:01.553 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (2m) 26s ago 24m 57.9M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:07:01.553 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (23m) 100s ago 23m 79.1M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:07:01.553 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (22m) 26s ago 22m 80.3M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:07:01.850 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:01 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:01.850 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:01 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:02.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:01 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:02.258 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-06-18T03:07:02.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:02 smithi163 ceph-mon[29791]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:07:02.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:02 smithi163 ceph-mon[29791]: from='client.15678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:02 smithi163 ceph-mon[25094]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:07:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:02 smithi163 ceph-mon[25094]: from='client.15678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:02 smithi186 ceph-mon[67751]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:07:03.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:02 smithi186 ceph-mon[67751]: from='client.15678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:04.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:07:03] "GET /metrics HTTP/1.1" 200 38007 "" "Prometheus/2.43.0" 2024-06-18T03:07:04.299 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:07:04.881 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:04 smithi163 ceph-mon[29791]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:04.882 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:04 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2746070848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:04.882 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:04 smithi163 ceph-mon[25094]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:04.882 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:04 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2746070848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:04.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade status' 2024-06-18T03:07:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:04 smithi186 ceph-mon[67751]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:04 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/2746070848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:06.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:06 smithi163 ceph-mon[29791]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:07:06.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:06 smithi163 ceph-mon[25094]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:07:06.934 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:07:06.934 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": null, 2024-06-18T03:07:06.934 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": false, 2024-06-18T03:07:06.934 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "", 2024-06-18T03:07:06.934 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:07:06.934 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": null, 2024-06-18T03:07:06.934 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "", 2024-06-18T03:07:06.934 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:07:06.934 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:07:07.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:06 smithi186 ceph-mon[67751]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:07:07.636 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph health detail' 2024-06-18T03:07:08.941 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:08 smithi163 ceph-mon[29791]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:08.941 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:08 smithi163 ceph-mon[29791]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:07:08.941 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:08 smithi163 ceph-mon[25094]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:08.941 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:08 smithi163 ceph-mon[25094]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:07:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:08 smithi186 ceph-mon[67751]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:08 smithi186 ceph-mon[67751]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:07:09.781 INFO:teuthology.orchestra.run.smithi163.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:07:09.782 INFO:teuthology.orchestra.run.smithi163.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:07:09.782 INFO:teuthology.orchestra.run.smithi163.stdout: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in unknown state 2024-06-18T03:07:10.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:09 smithi163 ceph-mon[25094]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:10.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:09 smithi163 ceph-mon[29791]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:10.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:09 smithi186 ceph-mon[67751]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:10.358 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-06-18T03:07:11.004 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:10 smithi163 ceph-mon[25094]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:11.004 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:10 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/2927636977' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:07:11.004 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:10 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:11.004 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:10 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/2927636977' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:07:11.033 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:10 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:11.033 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:10 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/2927636977' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:07:11.943 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:11 smithi163 ceph-mon[29791]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:07:11.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:11 smithi163 ceph-mon[25094]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:07:12.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:11 smithi186 ceph-mon[67751]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:07:12.935 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:12 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:07:12.935 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:12 smithi163 ceph-mon[29791]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:12.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:12 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:07:12.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:12 smithi163 ceph-mon[25094]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:13.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:12 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:07:13.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:12 smithi186 ceph-mon[67751]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:13.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:13 smithi163 ceph-mon[25094]: from='client.15708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "mon", "hosts": "smithi163", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:13.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:13 smithi163 ceph-mon[25094]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:13.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:13 smithi163 ceph-mon[29791]: from='client.15708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "mon", "hosts": "smithi163", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:13.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:13 smithi163 ceph-mon[29791]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:13 smithi186 ceph-mon[67751]: from='client.15708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "mon", "hosts": "smithi163", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:13 smithi186 ceph-mon[67751]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:14.168 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:07:13] "GET /metrics HTTP/1.1" 200 38004 "" "Prometheus/2.43.0" 2024-06-18T03:07:14.202 INFO:teuthology.orchestra.run.smithi163.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:07:14.853 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-18T03:07:15.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[25094]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:07:15.415 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:15.415 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:15.415 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:07:15.415 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[25094]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 0 op/s 2024-06-18T03:07:15.415 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[25094]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:07:15.415 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:15.415 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[25094]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:07:15.415 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[29791]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:07:15.415 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:15.415 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:15.415 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:07:15.415 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[29791]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 0 op/s 2024-06-18T03:07:15.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[29791]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:07:15.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:15.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:15 smithi163 ceph-mon[29791]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:07:15.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:15 smithi186 ceph-mon[67751]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:07:15.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:15 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:15.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:15 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:15 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:07:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:15 smithi186 ceph-mon[67751]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 0 op/s 2024-06-18T03:07:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:15 smithi186 ceph-mon[67751]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:07:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:15 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:15 smithi186 ceph-mon[67751]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:07:17.397 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:17.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[25094]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:07:17.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[25094]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:07:17.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:07:17.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:17.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[25094]: Upgrade: Setting container_image for all mgr 2024-06-18T03:07:17.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:17.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-18T03:07:17.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-06-18T03:07:17.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[25094]: Upgrade: It appears safe to stop mon.a 2024-06-18T03:07:17.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[25094]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:07:17.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:17.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[29791]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:07:17.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[29791]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:07:17.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:07:17.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:17.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[29791]: Upgrade: Setting container_image for all mgr 2024-06-18T03:07:17.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:17.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-18T03:07:17.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-06-18T03:07:17.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[29791]: Upgrade: It appears safe to stop mon.a 2024-06-18T03:07:17.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:17 smithi163 ceph-mon[29791]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:07:17.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:17 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:17.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:17 smithi186 ceph-mon[67751]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:07:17.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:17 smithi186 ceph-mon[67751]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:07:17.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:17 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:07:17.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:17 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:17.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:17 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mgr 2024-06-18T03:07:17.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:17 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:17.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:17 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-18T03:07:17.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:17 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-06-18T03:07:17.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:17 smithi186 ceph-mon[67751]: Upgrade: It appears safe to stop mon.a 2024-06-18T03:07:17.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:17 smithi186 ceph-mon[67751]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:07:18.763 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:07:19.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[29791]: Upgrade: Updating mon.a 2024-06-18T03:07:19.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:19.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:07:19.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:07:19.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:19.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[29791]: Deploying daemon mon.a on smithi163 2024-06-18T03:07:19.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[29791]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:07:19.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[25094]: Upgrade: Updating mon.a 2024-06-18T03:07:19.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:19.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:07:19.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:07:19.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[25094]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:19.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[25094]: Deploying daemon mon.a on smithi163 2024-06-18T03:07:19.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:19 smithi163 ceph-mon[25094]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:07:19.423 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:07:19.424 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (2m) 118s ago 23m 16.2M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:07:19.424 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (98s) 44s ago 23m 76.5M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:07:19.424 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 unknown 118s ago 23m 57.1M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:07:19.424 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (3m) 44s ago 28m 441M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:07:19.424 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (12m) 118s ago 30m 497M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:07:19.424 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (30m) 118s ago 30m 85.6M 2048M 17.2.0 e1d6a67b021e 60c2a1067068 2024-06-18T03:07:19.424 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (48s) 44s ago 28m 21.5M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:07:19.424 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (28m) 118s ago 28m 61.2M 2048M 17.2.0 e1d6a67b021e 158231a62b0c 2024-06-18T03:07:19.424 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (2m) 118s ago 24m 8837k - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:07:19.424 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (2m) 44s ago 24m 15.8M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:07:19.424 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (28m) 118s ago 28m 55.8M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T03:07:19.424 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (27m) 118s ago 27m 56.5M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T03:07:19.424 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (27m) 118s ago 27m 51.1M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:07:19.425 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (27m) 118s ago 27m 54.8M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:07:19.425 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (26m) 44s ago 26m 61.5M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:07:19.425 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (26m) 44s ago 26m 52.4M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:07:19.425 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (25m) 44s ago 25m 52.2M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:07:19.425 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (25m) 44s ago 25m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:07:19.425 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (2m) 44s ago 24m 57.9M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:07:19.425 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (23m) 118s ago 23m 79.1M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:07:19.425 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (23m) 44s ago 23m 80.3M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:07:19.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:19 smithi186 ceph-mon[67751]: Upgrade: Updating mon.a 2024-06-18T03:07:19.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:19 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' 2024-06-18T03:07:19.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:19 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:07:19.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:19 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:07:19.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:19 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:19.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:19 smithi186 ceph-mon[67751]: Deploying daemon mon.a on smithi163 2024-06-18T03:07:19.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:19 smithi186 ceph-mon[67751]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:07:19.826 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:07:19.826 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:07:19.826 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-06-18T03:07:19.826 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 1 2024-06-18T03:07:19.826 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:07:19.826 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:07:19.826 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:07:19.826 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:07:19.827 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:07:19.827 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-18T03:07:19.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:07:19.827 INFO:teuthology.orchestra.run.smithi163.stdout: "mds": {}, 2024-06-18T03:07:19.827 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:07:19.827 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:07:19.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:07:19.827 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:07:19.827 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-06-18T03:07:19.827 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:07:19.827 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:07:19.827 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:07:20.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:19 smithi163 systemd[1]: Stopping Ceph mon.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:07:20.191 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:07:20.191 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:07:20.191 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:07:20.191 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi163", 2024-06-18T03:07:20.191 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:07:20.191 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "0/2 daemons upgraded", 2024-06-18T03:07:20.191 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading mon daemons", 2024-06-18T03:07:20.192 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:07:20.192 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:07:20.335 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:07:20] ENGINE Bus STOPPING 2024-06-18T03:07:20.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-mon[29791]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:20.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-mon[29791]: from='client.15720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:20.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-mon[29791]: from='client.15726 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:20.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-mon[29791]: from='client.? 172.21.15.163:0/1929176' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:20.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-mon[25094]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:20.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-mon[25094]: from='client.15720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:20.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-mon[25094]: from='client.15726 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:20.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-mon[25094]: from='client.? 172.21.15.163:0/1929176' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:20.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a[25090]: 2024-06-18T03:07:20.139+0000 7fc407dce700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:07:20.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a[25090]: 2024-06-18T03:07:20.139+0000 7fc407dce700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-06-18T03:07:20.529 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:20 smithi186 ceph-mon[67751]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:20.529 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:20 smithi186 ceph-mon[67751]: from='client.15720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:20.529 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:20 smithi186 ceph-mon[67751]: from='client.15726 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:20.529 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:20 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/1929176' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:20.650 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:20 smithi163 podman[82705]: 2024-06-18 03:07:20.335302871 +0000 UTC m=+0.296111221 container died 60c2a10670680cb82ecf9c2b90540ea3d70fe1b12e7de6d6b36ee87c71b5d3f3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, ceph=True, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vendor=Red Hat, Inc., io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vcs-type=git, name=centos-stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-06-18T03:07:20.650 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:07:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-18T03:07:20.650 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:07:20] ENGINE Bus STOPPED 2024-06-18T03:07:20.651 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:07:20] ENGINE Bus STARTING 2024-06-18T03:07:20.651 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:07:20] ENGINE Serving on http://:::9283 2024-06-18T03:07:20.651 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:20 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:07:20] ENGINE Bus STARTED 2024-06-18T03:07:20.958 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:20 smithi163 podman[82705]: 2024-06-18 03:07:20.741014947 +0000 UTC m=+0.701823297 container cleanup 60c2a10670680cb82ecf9c2b90540ea3d70fe1b12e7de6d6b36ee87c71b5d3f3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, release=754, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-06-18T03:07:20.959 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:20 smithi163 bash[82705]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a 2024-06-18T03:07:21.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:20 smithi163 podman[82721]: 2024-06-18 03:07:20.95904434 +0000 UTC m=+0.621102658 container remove 60c2a10670680cb82ecf9c2b90540ea3d70fe1b12e7de6d6b36ee87c71b5d3f3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, GIT_CLEAN=True, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, ceph=True, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, distribution-scope=public, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container) 2024-06-18T03:07:21.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.a.service: Deactivated successfully. 2024-06-18T03:07:21.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 systemd[1]: Stopped Ceph mon.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:07:21.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.a.service: Consumed 25.498s CPU time. 2024-06-18T03:07:21.687 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 systemd[1]: Starting Ceph mon.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:07:21.687 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 podman[82860]: 2024-06-18 03:07:21.487871549 +0000 UTC m=+0.017439008 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:07:21.687 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 podman[82860]: 2024-06-18 03:07:21.585155621 +0000 UTC m=+0.114723071 container create 07c33ce786ba407922a5e24ccec20cedb7b1c687edf7795de0bf4ab41b14f6a9 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-06-18T03:07:22.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 podman[82860]: 2024-06-18 03:07:21.79625789 +0000 UTC m=+0.325825347 container init 07c33ce786ba407922a5e24ccec20cedb7b1c687edf7795de0bf4ab41b14f6a9 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0) 2024-06-18T03:07:22.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 podman[82860]: 2024-06-18 03:07:21.799789915 +0000 UTC m=+0.329357363 container start 07c33ce786ba407922a5e24ccec20cedb7b1c687edf7795de0bf4ab41b14f6a9 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-06-18T03:07:22.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: set uid:gid to 167:167 (ceph:ceph) 2024-06-18T03:07:22.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc), process ceph-mon, pid 2 2024-06-18T03:07:22.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: pidfile_write: ignore empty --pid-file 2024-06-18T03:07:22.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: load: jerasure load: lrc 2024-06-18T03:07:22.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: RocksDB version: 7.9.2 2024-06-18T03:07:22.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Git sha 0 2024-06-18T03:07:22.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Compile date 2024-06-17 15:21:02 2024-06-18T03:07:22.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: DB SUMMARY 2024-06-18T03:07:22.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: DB Session ID: PVQ9X09JQZ8H8F92F8TG 2024-06-18T03:07:22.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: CURRENT file: CURRENT 2024-06-18T03:07:22.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: IDENTITY file: IDENTITY 2024-06-18T03:07:22.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: MANIFEST file: MANIFEST-000015 size: 2889 Bytes 2024-06-18T03:07:22.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000060.sst 2024-06-18T03:07:22.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000058.log size: 2389666 ; 2024-06-18T03:07:22.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.error_if_exists: 0 2024-06-18T03:07:22.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.create_if_missing: 0 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.paranoid_checks: 1 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.flush_verify_memtable_count: 1 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.env: 0x55af94859740 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.fs: PosixFileSystem 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.info_log: 0x55af97067000 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_file_opening_threads: 16 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.statistics: (nil) 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.use_fsync: 0 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_log_file_size: 0 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.keep_log_file_num: 1000 2024-06-18T03:07:22.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.recycle_log_file_num: 0 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.allow_fallocate: 1 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.allow_mmap_reads: 0 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.allow_mmap_writes: 0 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.use_direct_reads: 0 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.create_missing_column_families: 0 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.db_log_dir: 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.wal_dir: 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-18T03:07:22.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.advise_random_on_open: 1 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.db_write_buffer_size: 0 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.write_buffer_manager: 0x55af97076c80 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.rate_limiter: (nil) 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.wal_recovery_mode: 2 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.enable_thread_tracking: 0 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.enable_pipelined_write: 0 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.unordered_write: 0 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-18T03:07:22.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-18T03:07:22.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-18T03:07:22.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-18T03:07:22.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.row_cache: None 2024-06-18T03:07:22.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.wal_filter: None 2024-06-18T03:07:22.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-18T03:07:22.299 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.allow_ingest_behind: 0 2024-06-18T03:07:22.299 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.two_write_queues: 0 2024-06-18T03:07:22.299 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.manual_wal_flush: 0 2024-06-18T03:07:22.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.wal_compression: 0 2024-06-18T03:07:22.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.atomic_flush: 0 2024-06-18T03:07:22.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-18T03:07:22.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-18T03:07:22.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-18T03:07:22.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.log_readahead_size: 0 2024-06-18T03:07:22.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-18T03:07:22.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.best_efforts_recovery: 0 2024-06-18T03:07:22.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-18T03:07:22.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-18T03:07:22.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.allow_data_in_errors: 0 2024-06-18T03:07:22.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.db_host_id: __hostname__ 2024-06-18T03:07:22.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.enforce_single_del_contracts: true 2024-06-18T03:07:22.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_background_jobs: 2 2024-06-18T03:07:22.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_background_compactions: -1 2024-06-18T03:07:22.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_subcompactions: 1 2024-06-18T03:07:22.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-18T03:07:22.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-18T03:07:22.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-18T03:07:22.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_total_wal_size: 0 2024-06-18T03:07:22.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-18T03:07:22.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-18T03:07:22.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-18T03:07:22.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-18T03:07:22.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_open_files: -1 2024-06-18T03:07:22.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.bytes_per_sync: 0 2024-06-18T03:07:22.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-18T03:07:22.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-18T03:07:22.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compaction_readahead_size: 0 2024-06-18T03:07:22.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_background_flushes: -1 2024-06-18T03:07:22.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Compression algorithms supported: 2024-06-18T03:07:22.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: kZSTD supported: 0 2024-06-18T03:07:22.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: kXpressCompression supported: 0 2024-06-18T03:07:22.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: kBZip2Compression supported: 0 2024-06-18T03:07:22.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-18T03:07:22.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: kLZ4Compression supported: 1 2024-06-18T03:07:22.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: kZlibCompression supported: 1 2024-06-18T03:07:22.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: kLZ4HCCompression supported: 1 2024-06-18T03:07:22.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: kSnappyCompression supported: 1 2024-06-18T03:07:22.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-18T03:07:22.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: DMutex implementation: pthread_mutex_t 2024-06-18T03:07:22.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-06-18T03:07:22.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-06-18T03:07:22.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-18T03:07:22.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.merge_operator: 2024-06-18T03:07:22.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compaction_filter: None 2024-06-18T03:07:22.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compaction_filter_factory: None 2024-06-18T03:07:22.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.sst_partitioner_factory: None 2024-06-18T03:07:22.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-18T03:07:22.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-18T03:07:22.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55af97066d00) 2024-06-18T03:07:22.304 INFO:journalctl@ceph.mon.a.smithi163.stdout: cache_index_and_filter_blocks: 1 2024-06-18T03:07:22.304 INFO:journalctl@ceph.mon.a.smithi163.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-18T03:07:22.304 INFO:journalctl@ceph.mon.a.smithi163.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-18T03:07:22.304 INFO:journalctl@ceph.mon.a.smithi163.stdout: pin_top_level_index_and_filter: 1 2024-06-18T03:07:22.304 INFO:journalctl@ceph.mon.a.smithi163.stdout: index_type: 0 2024-06-18T03:07:22.304 INFO:journalctl@ceph.mon.a.smithi163.stdout: data_block_index_type: 0 2024-06-18T03:07:22.304 INFO:journalctl@ceph.mon.a.smithi163.stdout: index_shortening: 1 2024-06-18T03:07:22.304 INFO:journalctl@ceph.mon.a.smithi163.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-18T03:07:22.304 INFO:journalctl@ceph.mon.a.smithi163.stdout: checksum: 4 2024-06-18T03:07:22.304 INFO:journalctl@ceph.mon.a.smithi163.stdout: no_block_cache: 0 2024-06-18T03:07:22.304 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache: 0x55af9705d090 2024-06-18T03:07:22.304 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache_name: BinnedLRUCache 2024-06-18T03:07:22.305 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache_options: 2024-06-18T03:07:22.305 INFO:journalctl@ceph.mon.a.smithi163.stdout: capacity : 536870912 2024-06-18T03:07:22.305 INFO:journalctl@ceph.mon.a.smithi163.stdout: num_shard_bits : 4 2024-06-18T03:07:22.305 INFO:journalctl@ceph.mon.a.smithi163.stdout: strict_capacity_limit : 0 2024-06-18T03:07:22.305 INFO:journalctl@ceph.mon.a.smithi163.stdout: high_pri_pool_ratio: 0.000 2024-06-18T03:07:22.305 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache_compressed: (nil) 2024-06-18T03:07:22.305 INFO:journalctl@ceph.mon.a.smithi163.stdout: persistent_cache: (nil) 2024-06-18T03:07:22.305 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_size: 4096 2024-06-18T03:07:22.305 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_size_deviation: 10 2024-06-18T03:07:22.305 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_restart_interval: 16 2024-06-18T03:07:22.305 INFO:journalctl@ceph.mon.a.smithi163.stdout: index_block_restart_interval: 1 2024-06-18T03:07:22.305 INFO:journalctl@ceph.mon.a.smithi163.stdout: metadata_block_size: 4096 2024-06-18T03:07:22.305 INFO:journalctl@ceph.mon.a.smithi163.stdout: partition_filters: 0 2024-06-18T03:07:22.306 INFO:journalctl@ceph.mon.a.smithi163.stdout: use_delta_encoding: 1 2024-06-18T03:07:22.306 INFO:journalctl@ceph.mon.a.smithi163.stdout: filter_policy: bloomfilter 2024-06-18T03:07:22.306 INFO:journalctl@ceph.mon.a.smithi163.stdout: whole_key_filtering: 1 2024-06-18T03:07:22.306 INFO:journalctl@ceph.mon.a.smithi163.stdout: verify_compression: 0 2024-06-18T03:07:22.306 INFO:journalctl@ceph.mon.a.smithi163.stdout: read_amp_bytes_per_bit: 0 2024-06-18T03:07:22.306 INFO:journalctl@ceph.mon.a.smithi163.stdout: format_version: 5 2024-06-18T03:07:22.306 INFO:journalctl@ceph.mon.a.smithi163.stdout: enable_index_compression: 1 2024-06-18T03:07:22.306 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_align: 0 2024-06-18T03:07:22.306 INFO:journalctl@ceph.mon.a.smithi163.stdout: max_auto_readahead_size: 262144 2024-06-18T03:07:22.306 INFO:journalctl@ceph.mon.a.smithi163.stdout: prepopulate_block_cache: 0 2024-06-18T03:07:22.306 INFO:journalctl@ceph.mon.a.smithi163.stdout: initial_auto_readahead_size: 8192 2024-06-18T03:07:22.306 INFO:journalctl@ceph.mon.a.smithi163.stdout: num_file_reads_for_auto_readahead: 2 2024-06-18T03:07:22.306 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.write_buffer_size: 33554432 2024-06-18T03:07:22.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_write_buffer_number: 2 2024-06-18T03:07:22.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compression: NoCompression 2024-06-18T03:07:22.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.bottommost_compression: Disabled 2024-06-18T03:07:22.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.prefix_extractor: nullptr 2024-06-18T03:07:22.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-18T03:07:22.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.num_levels: 7 2024-06-18T03:07:22.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-18T03:07:22.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-18T03:07:22.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-18T03:07:22.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-18T03:07:22.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-18T03:07:22.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-18T03:07:22.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compression_opts.level: 32767 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compression_opts.strategy: 0 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compression_opts.enabled: false 2024-06-18T03:07:22.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-06-18T03:07:22.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-18T03:07:22.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-18T03:07:22.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-18T03:07:22.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.target_file_size_base: 67108864 2024-06-18T03:07:22.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-18T03:07:22.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-18T03:07:22.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-18T03:07:22.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-18T03:07:22.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-18T03:07:22.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-18T03:07:22.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-18T03:07:22.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-18T03:07:22.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-18T03:07:22.310 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-18T03:07:22.310 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-18T03:07:22.310 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-18T03:07:22.310 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-18T03:07:22.310 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-06-18T03:07:22.310 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.arena_block_size: 1048576 2024-06-18T03:07:22.310 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-18T03:07:22.310 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-18T03:07:22.310 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.disable_auto_compactions: 0 2024-06-18T03:07:22.310 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-18T03:07:22.310 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-18T03:07:22.310 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-18T03:07:22.310 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.table_properties_collectors: 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.inplace_update_support: 0 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.bloom_locality: 0 2024-06-18T03:07:22.311 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.max_successive_merges: 0 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.paranoid_file_checks: 0 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.force_consistency_checks: 1 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.report_bg_io_stats: 0 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.ttl: 2592000 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.enable_blob_files: false 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.min_blob_size: 0 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.blob_file_size: 268435456 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-18T03:07:22.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.blob_file_starting_level: 0 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 62, last_sequence is 26766, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: b39d3d03-9637-4565-bdeb-ab6a9323dbaa 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718680041826909, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718680041834456, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 2186545, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 26760, "largest_seqno": 28899, "table_properties": {"data_size": 2178296, "index_size": 4925, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2309, "raw_key_size": 22752, "raw_average_key_size": 24, "raw_value_size": 2159470, "raw_average_value_size": 2344, "num_data_blocks": 222, "num_entries": 921, "num_filter_entries": 921, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1718680041, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "b39d3d03-9637-4565-bdeb-ab6a9323dbaa", "db_session_id": "PVQ9X09JQZ8H8F92F8TG", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718680041834562, "job": 1, "event": "recovery_finished"} 2024-06-18T03:07:22.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-06-18T03:07:22.314 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-18T03:07:22.314 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-18T03:07:22.314 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55af9708ae00 2024-06-18T03:07:22.314 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: DB pointer 0x55af97180000 2024-06-18T03:07:22.314 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-06-18T03:07:22.314 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-06-18T03:07:22.314 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** DB Stats ** 2024-06-18T03:07:22.314 INFO:journalctl@ceph.mon.a.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-18T03:07:22.314 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-18T03:07:22.314 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-18T03:07:22.314 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-18T03:07:22.314 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-18T03:07:22.314 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-18T03:07:22.315 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-18T03:07:22.315 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2024-06-18T03:07:22.315 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** Compaction Stats [default] ** 2024-06-18T03:07:22.315 INFO:journalctl@ceph.mon.a.smithi163.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-18T03:07:22.315 INFO:journalctl@ceph.mon.a.smithi163.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-06-18T03:07:22.315 INFO:journalctl@ceph.mon.a.smithi163.stdout: L0 1/0 2.09 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 572.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-18T03:07:22.315 INFO:journalctl@ceph.mon.a.smithi163.stdout: L6 1/0 9.11 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-06-18T03:07:22.315 INFO:journalctl@ceph.mon.a.smithi163.stdout: Sum 2/0 11.19 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 572.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-18T03:07:22.315 INFO:journalctl@ceph.mon.a.smithi163.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 572.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-18T03:07:22.315 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2024-06-18T03:07:22.315 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** Compaction Stats [default] ** 2024-06-18T03:07:22.315 INFO:journalctl@ceph.mon.a.smithi163.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-18T03:07:22.315 INFO:journalctl@ceph.mon.a.smithi163.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 572.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative compaction: 0.00 GB write, 163.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval compaction: 0.00 GB write, 163.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-18T03:07:22.316 INFO:journalctl@ceph.mon.a.smithi163.stdout: Block cache BinnedLRUCache@0x55af9705d090#2 capacity: 512.00 MB usage: 7.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-06-18T03:07:22.317 INFO:journalctl@ceph.mon.a.smithi163.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.53 KB,0.000482798%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-06-18T03:07:22.317 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2024-06-18T03:07:22.317 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-18T03:07:22.317 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: starting mon.a rank 0 at public addrs [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] at bind addrs [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T03:07:22.317 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: mon.a@-1(???) e3 preinit fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T03:07:22.317 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: mon.a@-1(???).mds e1 new map 2024-06-18T03:07:22.318 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: mon.a@-1(???).mds e1 print_map 2024-06-18T03:07:22.318 INFO:journalctl@ceph.mon.a.smithi163.stdout: e1 2024-06-18T03:07:22.318 INFO:journalctl@ceph.mon.a.smithi163.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-18T03:07:22.318 INFO:journalctl@ceph.mon.a.smithi163.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-18T03:07:22.318 INFO:journalctl@ceph.mon.a.smithi163.stdout: legacy client fscid: -1 2024-06-18T03:07:22.318 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2024-06-18T03:07:22.318 INFO:journalctl@ceph.mon.a.smithi163.stdout: No filesystems configured 2024-06-18T03:07:22.318 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: mon.a@-1(???).osd e111 crush map has features 3314933000854323200, adjusting msgr requires 2024-06-18T03:07:22.318 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: mon.a@-1(???).osd e111 crush map has features 432629239337189376, adjusting msgr requires 2024-06-18T03:07:22.318 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: mon.a@-1(???).osd e111 crush map has features 432629239337189376, adjusting msgr requires 2024-06-18T03:07:22.319 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: mon.a@-1(???).osd e111 crush map has features 432629239337189376, adjusting msgr requires 2024-06-18T03:07:22.319 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 ceph-mon[82879]: mon.a@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-06-18T03:07:22.319 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 bash[82860]: 07c33ce786ba407922a5e24ccec20cedb7b1c687edf7795de0bf4ab41b14f6a9 2024-06-18T03:07:22.319 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:21 smithi163 systemd[1]: Started Ceph mon.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:07:23.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: ignoring --setuser ceph since I am not root 2024-06-18T03:07:23.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: ignoring --setgroup ceph since I am not root 2024-06-18T03:07:23.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mgr[60816]: -- 172.21.15.186:0/1995496638 <== mon.2 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5635f991f0e0 con 0x5635f98fd400 2024-06-18T03:07:23.569 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:23.426+0000 7f55e1706040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T03:07:23.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:07:23.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:07:23.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:07:23.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: from='client.15738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:23.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:07:23.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-06-18T03:07:23.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: mon.a calling monitor election 2024-06-18T03:07:23.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-18T03:07:23.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: monmap epoch 3 2024-06-18T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: last_changed 2024-06-18T02:38:33.877201+0000 2024-06-18T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: created 2024-06-18T02:36:57.402480+0000 2024-06-18T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: min_mon_release 17 (quincy) 2024-06-18T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: election_strategy: 1 2024-06-18T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: 0: [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] mon.a 2024-06-18T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: 1: [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] mon.c 2024-06-18T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: 2: [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon.b 2024-06-18T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: fsmap 2024-06-18T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: osdmap e111: 8 total, 8 up, 8 in 2024-06-18T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: mgrmap e41: y(active, since 3m), standbys: x 2024-06-18T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:07:23.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in unknown state 2024-06-18T03:07:23.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: mgrmap e42: y(active, since 3m), standbys: x 2024-06-18T03:07:23.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-mon[67751]: from='mgr.15321 ' entity='' 2024-06-18T03:07:23.577 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ignoring --setuser ceph since I am not root 2024-06-18T03:07:23.578 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ignoring --setgroup ceph since I am not root 2024-06-18T03:07:23.578 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mgr[62744]: -- 172.21.15.163:0/3800622821 <== mon.0 v2:172.21.15.163:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5573ea06d0e0 con 0x5573ea04b400 2024-06-18T03:07:23.578 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:23.427+0000 7f1104977040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-18T03:07:23.578 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:23.575+0000 7f1104977040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T03:07:23.578 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:07:23.578 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:07:23.578 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:07:23.578 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: from='client.15738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:23.578 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:07:23.578 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: mon.a calling monitor election 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: monmap epoch 3 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: last_changed 2024-06-18T02:38:33.877201+0000 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: created 2024-06-18T02:36:57.402480+0000 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: min_mon_release 17 (quincy) 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: election_strategy: 1 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: 0: [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] mon.a 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: 1: [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] mon.c 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: 2: [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon.b 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: fsmap 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: osdmap e111: 8 total, 8 up, 8 in 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: mgrmap e41: y(active, since 3m), standbys: x 2024-06-18T03:07:23.579 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:07:23.580 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:07:23.580 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in unknown state 2024-06-18T03:07:23.580 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: mgrmap e42: y(active, since 3m), standbys: x 2024-06-18T03:07:23.580 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[29791]: from='mgr.15321 ' entity='' 2024-06-18T03:07:23.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:07:23.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:07:23.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: from='mgr.15321 172.21.15.163:0/437461030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:07:23.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: from='client.15738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:23.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:07:23.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-06-18T03:07:23.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: mon.a calling monitor election 2024-06-18T03:07:23.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-18T03:07:23.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: monmap epoch 3 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: last_changed 2024-06-18T02:38:33.877201+0000 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: created 2024-06-18T02:36:57.402480+0000 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: min_mon_release 17 (quincy) 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: election_strategy: 1 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: 0: [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] mon.a 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: 1: [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] mon.c 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: 2: [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon.b 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: fsmap 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: osdmap e111: 8 total, 8 up, 8 in 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: mgrmap e41: y(active, since 3m), standbys: x 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in unknown state 2024-06-18T03:07:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: mgrmap e42: y(active, since 3m), standbys: x 2024-06-18T03:07:23.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-mon[82879]: from='mgr.15321 ' entity='' 2024-06-18T03:07:23.890 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:23.645+0000 7f1104977040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T03:07:23.893 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:23.578+0000 7f55e1706040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-18T03:07:23.893 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:23.646+0000 7f55e1706040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-18T03:07:24.242 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:23.888+0000 7f1104977040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T03:07:24.242 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:23.950+0000 7f1104977040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T03:07:24.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:23.892+0000 7f55e1706040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-18T03:07:24.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:23.954+0000 7f55e1706040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-18T03:07:24.542 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:07:24 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:07:24.242Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:07:24.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:07:24 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:07:24.243Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:07:25.267 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:24 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:24.892+0000 7f1104977040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T03:07:25.268 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:25.009+0000 7f1104977040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T03:07:25.268 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:25.067+0000 7f1104977040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T03:07:25.268 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:24 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:24.903+0000 7f55e1706040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-18T03:07:25.268 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:25 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:25.020+0000 7f55e1706040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-18T03:07:25.268 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:25 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:25.078+0000 7f55e1706040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-18T03:07:25.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:25 smithi163 ceph-mon[29791]: mgrmap e43: y(active, since 3m), standbys: x 2024-06-18T03:07:25.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:25 smithi163 ceph-mon[82879]: mgrmap e43: y(active, since 3m), standbys: x 2024-06-18T03:07:25.543 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:25.285+0000 7f1104977040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T03:07:25.543 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:25.347+0000 7f1104977040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T03:07:25.543 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:25.416+0000 7f1104977040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T03:07:25.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:25 smithi186 ceph-mon[67751]: mgrmap e43: y(active, since 3m), standbys: x 2024-06-18T03:07:25.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:25 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:25.296+0000 7f55e1706040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-18T03:07:25.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:25 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:25.359+0000 7f55e1706040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-18T03:07:25.568 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:25 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:25.428+0000 7f55e1706040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-18T03:07:25.994 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:25.566+0000 7f1104977040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T03:07:26.004 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:25 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:25.579+0000 7f55e1706040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-18T03:07:26.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:25.992+0000 7f1104977040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T03:07:26.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:26 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:26.004+0000 7f55e1706040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-18T03:07:26.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:26.317+0000 7f1104977040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T03:07:26.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:26.384+0000 7f1104977040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T03:07:26.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:26.536+0000 7f1104977040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-18T03:07:26.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:26 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:26.327+0000 7f55e1706040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-18T03:07:26.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:26 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:26.395+0000 7f55e1706040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-18T03:07:26.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:26 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:26.547+0000 7f55e1706040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-18T03:07:27.111 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:26.854+0000 7f1104977040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T03:07:27.111 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:26.920+0000 7f1104977040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T03:07:27.128 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:26 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:26.872+0000 7f55e1706040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-18T03:07:27.128 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:26 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:26.939+0000 7f55e1706040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-18T03:07:27.426 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:27 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:27.127+0000 7f55e1706040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T03:07:27.427 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:27 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:27.276+0000 7f55e1706040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T03:07:27.427 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:27 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:27.340+0000 7f55e1706040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T03:07:27.427 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:27 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:27.426+0000 7f55e1706040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T03:07:27.430 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:27.109+0000 7f1104977040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-18T03:07:27.431 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:27.268+0000 7f1104977040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-18T03:07:27.431 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:27.337+0000 7f1104977040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-18T03:07:27.431 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:27.429+0000 7f1104977040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-18T03:07:27.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:27 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:27.494+0000 7f1104977040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T03:07:27.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:27 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:27.486+0000 7f55e1706040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-18T03:07:28.300 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-mon[67751]: Standby manager daemon x restarted 2024-06-18T03:07:28.300 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-mon[67751]: Standby manager daemon x started 2024-06-18T03:07:28.300 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-mon[67751]: from='mgr.? 172.21.15.186:0/1205734021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T03:07:28.300 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-mon[67751]: from='mgr.? 172.21.15.186:0/1205734021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T03:07:28.301 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-mon[67751]: from='mgr.? 172.21.15.186:0/1205734021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T03:07:28.301 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-mon[67751]: from='mgr.? 172.21.15.186:0/1205734021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T03:07:28.301 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-mon[67751]: Active manager daemon y restarted 2024-06-18T03:07:28.301 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-mon[67751]: Activating manager daemon y 2024-06-18T03:07:28.301 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:28.030+0000 7f55e1706040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T03:07:28.301 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:28.193+0000 7f55e1706040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T03:07:28.301 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: [18/Jun/2024:03:07:28] ENGINE Bus STARTING 2024-06-18T03:07:28.302 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: CherryPy Checker: 2024-06-18T03:07:28.302 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: The Application mounted at '' has an empty config. 2024-06-18T03:07:28.302 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: 2024-06-18T03:07:28.302 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: [18/Jun/2024:03:07:28] ENGINE Serving on http://:::9283 2024-06-18T03:07:28.302 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:07:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x[60812]: [18/Jun/2024:03:07:28] ENGINE Bus STARTED 2024-06-18T03:07:28.331 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:07:28.299Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=5 2024-06-18T03:07:28.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:28.068+0000 7f1104977040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-18T03:07:28.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:28.234+0000 7f1104977040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-18T03:07:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[29791]: Standby manager daemon x restarted 2024-06-18T03:07:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[29791]: Standby manager daemon x started 2024-06-18T03:07:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/1205734021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T03:07:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/1205734021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T03:07:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/1205734021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T03:07:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[29791]: from='mgr.? 172.21.15.186:0/1205734021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T03:07:28.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[29791]: Active manager daemon y restarted 2024-06-18T03:07:28.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[29791]: Activating manager daemon y 2024-06-18T03:07:28.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[82879]: Standby manager daemon x restarted 2024-06-18T03:07:28.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[82879]: Standby manager daemon x started 2024-06-18T03:07:28.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[82879]: from='mgr.? 172.21.15.186:0/1205734021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-18T03:07:28.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[82879]: from='mgr.? 172.21.15.186:0/1205734021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-18T03:07:28.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[82879]: from='mgr.? 172.21.15.186:0/1205734021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-18T03:07:28.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[82879]: from='mgr.? 172.21.15.186:0/1205734021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-18T03:07:28.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[82879]: Active manager daemon y restarted 2024-06-18T03:07:28.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-mon[82879]: Activating manager daemon y 2024-06-18T03:07:28.727 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:07:28] ENGINE Bus STARTING 2024-06-18T03:07:28.727 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: CherryPy Checker: 2024-06-18T03:07:28.727 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: The Application mounted at '' has an empty config. 2024-06-18T03:07:28.727 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:28.727 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:07:28] ENGINE Serving on http://:::9283 2024-06-18T03:07:28.727 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:07:28] ENGINE Bus STARTED 2024-06-18T03:07:29.513 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:29.273+0000 7f10a413f640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-06-18T03:07:29.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: osdmap e112: 8 total, 8 up, 8 in 2024-06-18T03:07:29.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: mgrmap e44: y(active, starting, since 0.0137846s), standbys: x 2024-06-18T03:07:29.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:07:29.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:07:29.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:07:29.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T03:07:29.513 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: Manager daemon y is now available 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:07:29.514 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: osdmap e112: 8 total, 8 up, 8 in 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: mgrmap e44: y(active, starting, since 0.0137846s), standbys: x 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T03:07:29.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T03:07:29.516 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T03:07:29.516 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T03:07:29.516 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T03:07:29.516 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T03:07:29.516 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T03:07:29.516 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: Manager daemon y is now available 2024-06-18T03:07:29.516 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:07:29.516 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T03:07:29.516 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T03:07:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: osdmap e112: 8 total, 8 up, 8 in 2024-06-18T03:07:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: mgrmap e44: y(active, starting, since 0.0137846s), standbys: x 2024-06-18T03:07:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:07:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:07:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:07:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: Manager daemon y is now available 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-18T03:07:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-18T03:07:30.274 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:29.804+0000 7f105c4a8640 -1 log_channel(cephadm) log [ERR] : [18/Jun/2024:03:07:29] ENGINE Error in HTTPServer.serve 2024-06-18T03:07:30.274 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: Traceback (most recent call last): 2024-06-18T03:07:30.274 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T03:07:30.274 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: self._connections.run(self.expiration_interval) 2024-06-18T03:07:30.274 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T03:07:30.274 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: self._run(expiration_interval) 2024-06-18T03:07:30.274 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T03:07:30.275 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T03:07:30.275 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T03:07:30.275 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T03:07:30.275 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T03:07:30.275 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: s = self.context.wrap_socket( 2024-06-18T03:07:30.275 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T03:07:30.275 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: return self.sslsocket_class._create( 2024-06-18T03:07:30.275 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T03:07:30.275 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: self.do_handshake() 2024-06-18T03:07:30.275 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T03:07:30.275 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: self._sslobj.do_handshake() 2024-06-18T03:07:30.275 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T03:07:30.275 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:29 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:30.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:30 smithi186 ceph-mon[67751]: mgrmap e45: y(active, since 1.03304s), standbys: x 2024-06-18T03:07:30.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:30 smithi186 ceph-mon[67751]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:07:30.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:30 smithi163 ceph-mon[29791]: mgrmap e45: y(active, since 1.03304s), standbys: x 2024-06-18T03:07:30.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:30 smithi163 ceph-mon[29791]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:07:30.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:30 smithi163 ceph-mon[82879]: mgrmap e45: y(active, since 1.03304s), standbys: x 2024-06-18T03:07:30.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:30 smithi163 ceph-mon[82879]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:07:31.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[82879]: [18/Jun/2024:03:07:29] ENGINE Bus STARTING 2024-06-18T03:07:31.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[82879]: [18/Jun/2024:03:07:29] ENGINE Serving on http://172.21.15.163:8765 2024-06-18T03:07:31.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:31.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[82879]: [18/Jun/2024:03:07:29] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T03:07:31.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[82879]: [18/Jun/2024:03:07:29] ENGINE Bus STARTED 2024-06-18T03:07:31.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[82879]: [18/Jun/2024:03:07:29] ENGINE Error in HTTPServer.serve 2024-06-18T03:07:31.543 INFO:journalctl@ceph.mon.a.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:07:31.543 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T03:07:31.543 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._connections.run(self.expiration_interval) 2024-06-18T03:07:31.543 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T03:07:31.543 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._run(expiration_interval) 2024-06-18T03:07:31.543 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout: s = self.context.wrap_socket( 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout: return self.sslsocket_class._create( 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout: self.do_handshake() 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout: self._sslobj.do_handshake() 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[82879]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:07:31.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[82879]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-06-18T03:07:31.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[82879]: Cluster is now healthy 2024-06-18T03:07:31.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[82879]: mgrmap e46: y(active, since 2s), standbys: x 2024-06-18T03:07:31.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[29791]: [18/Jun/2024:03:07:29] ENGINE Bus STARTING 2024-06-18T03:07:31.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[29791]: [18/Jun/2024:03:07:29] ENGINE Serving on http://172.21.15.163:8765 2024-06-18T03:07:31.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:31.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[29791]: [18/Jun/2024:03:07:29] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T03:07:31.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[29791]: [18/Jun/2024:03:07:29] ENGINE Bus STARTED 2024-06-18T03:07:31.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[29791]: [18/Jun/2024:03:07:29] ENGINE Error in HTTPServer.serve 2024-06-18T03:07:31.545 INFO:journalctl@ceph.mon.c.smithi163.stdout: Traceback (most recent call last): 2024-06-18T03:07:31.545 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T03:07:31.545 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._connections.run(self.expiration_interval) 2024-06-18T03:07:31.545 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._run(expiration_interval) 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: s = self.context.wrap_socket( 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: return self.sslsocket_class._create( 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: self.do_handshake() 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: self._sslobj.do_handshake() 2024-06-18T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi163.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T03:07:31.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[29791]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:07:31.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[29791]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-06-18T03:07:31.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[29791]: Cluster is now healthy 2024-06-18T03:07:31.547 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:31 smithi163 ceph-mon[29791]: mgrmap e46: y(active, since 2s), standbys: x 2024-06-18T03:07:31.563 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:31 smithi186 ceph-mon[67751]: [18/Jun/2024:03:07:29] ENGINE Bus STARTING 2024-06-18T03:07:31.563 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:31 smithi186 ceph-mon[67751]: [18/Jun/2024:03:07:29] ENGINE Serving on http://172.21.15.163:8765 2024-06-18T03:07:31.564 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:31 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:31.564 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:31 smithi186 ceph-mon[67751]: [18/Jun/2024:03:07:29] ENGINE Serving on https://172.21.15.163:7150 2024-06-18T03:07:31.564 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:31 smithi186 ceph-mon[67751]: [18/Jun/2024:03:07:29] ENGINE Bus STARTED 2024-06-18T03:07:31.564 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:31 smithi186 ceph-mon[67751]: [18/Jun/2024:03:07:29] ENGINE Error in HTTPServer.serve 2024-06-18T03:07:31.564 INFO:journalctl@ceph.mon.b.smithi186.stdout: Traceback (most recent call last): 2024-06-18T03:07:31.564 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-18T03:07:31.564 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._connections.run(self.expiration_interval) 2024-06-18T03:07:31.564 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-18T03:07:31.564 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._run(expiration_interval) 2024-06-18T03:07:31.564 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-18T03:07:31.564 INFO:journalctl@ceph.mon.b.smithi186.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-18T03:07:31.564 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout: s = self.context.wrap_socket( 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout: return self.sslsocket_class._create( 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout: self.do_handshake() 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout: self._sslobj.do_handshake() 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:31 smithi186 ceph-mon[67751]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:31 smithi186 ceph-mon[67751]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:31 smithi186 ceph-mon[67751]: Cluster is now healthy 2024-06-18T03:07:31.565 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:31 smithi186 ceph-mon[67751]: mgrmap e46: y(active, since 2s), standbys: x 2024-06-18T03:07:32.292 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:07:32 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:07:32.068Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=6 2024-06-18T03:07:33.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:33 smithi163 ceph-mon[29791]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:07:33.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:33 smithi163 ceph-mon[29791]: mgrmap e47: y(active, since 4s), standbys: x 2024-06-18T03:07:33.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:33 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:33.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:33 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:33.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:33 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:33.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:33 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:33.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:33 smithi163 ceph-mon[82879]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:07:33.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:33 smithi163 ceph-mon[82879]: mgrmap e47: y(active, since 4s), standbys: x 2024-06-18T03:07:33.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:33 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:33.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:33 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:33.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:33 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:33.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:33 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:33.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:33 smithi186 ceph-mon[67751]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:07:33.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:33 smithi186 ceph-mon[67751]: mgrmap e47: y(active, since 4s), standbys: x 2024-06-18T03:07:33.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:33 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:33.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:33 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:33.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:33 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:33.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:33 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:34.145 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:07:33] "GET /metrics HTTP/1.1" 200 35212 "" "Prometheus/2.43.0" 2024-06-18T03:07:34.428 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:34.428 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:34.428 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:34.428 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:34.428 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:34 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:34.428 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:34 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:34.428 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:34 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:34.429 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:34 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:34.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:34.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:34.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:34.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:35.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:35 smithi186 ceph-mon[67751]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:07:35.596 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:35 smithi163 ceph-mon[82879]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:07:35.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:35 smithi163 ceph-mon[29791]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:07:37.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[29791]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-18T03:07:37.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:37.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:37.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:37.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:37.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:37.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:07:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[82879]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-18T03:07:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:37 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:07:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:37 smithi186 ceph-mon[67751]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-18T03:07:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:37 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:37 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:37 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:37 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:37 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:37 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:07:38.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: Detected new or changed devices on smithi163 2024-06-18T03:07:38.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T03:07:38.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T03:07:38.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:07:38.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:07:38.575 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.575 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.575 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.575 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.575 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.575 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:07:38.575 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:38.575 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-18T03:07:38.575 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-06-18T03:07:38.575 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: Detected new or changed devices on smithi163 2024-06-18T03:07:38.575 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T03:07:38.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T03:07:38.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:07:38.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:07:38.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:07:38.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:38.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-18T03:07:38.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-06-18T03:07:38.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: Detected new or changed devices on smithi163 2024-06-18T03:07:38.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: Updating smithi163:/etc/ceph/ceph.conf 2024-06-18T03:07:38.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: Updating smithi186:/etc/ceph/ceph.conf 2024-06-18T03:07:38.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:07:38.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-18T03:07:38.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.721 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:38.721 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:07:38.721 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:38.721 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-18T03:07:38.721 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-06-18T03:07:39.692 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[29791]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:07:39.692 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[29791]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:07:39.692 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[29791]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:07:39.692 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[29791]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:07:39.692 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[29791]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-18T03:07:39.692 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[29791]: Upgrade: It appears safe to stop mon.c 2024-06-18T03:07:39.692 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[29791]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-06-18T03:07:39.693 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[82879]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:07:39.693 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[82879]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:07:39.693 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[82879]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:07:39.693 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[82879]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:07:39.693 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[82879]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-18T03:07:39.693 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[82879]: Upgrade: It appears safe to stop mon.c 2024-06-18T03:07:39.693 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:39 smithi163 ceph-mon[82879]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-06-18T03:07:39.732 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:39 smithi186 ceph-mon[67751]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:07:39.732 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:39 smithi186 ceph-mon[67751]: Updating smithi186:/etc/ceph/ceph.client.admin.keyring 2024-06-18T03:07:39.732 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:39 smithi186 ceph-mon[67751]: Updating smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:07:39.732 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:39 smithi186 ceph-mon[67751]: Updating smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-18T03:07:39.732 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:39 smithi186 ceph-mon[67751]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-18T03:07:39.732 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:39 smithi186 ceph-mon[67751]: Upgrade: It appears safe to stop mon.c 2024-06-18T03:07:39.732 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:39 smithi186 ceph-mon[67751]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-06-18T03:07:40.779 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:40 smithi163 ceph-mon[29791]: Upgrade: Updating mon.c 2024-06-18T03:07:40.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:40 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:40.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:40 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:07:40.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:40 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:07:40.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:40 smithi163 ceph-mon[29791]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:40.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:40 smithi163 ceph-mon[82879]: Upgrade: Updating mon.c 2024-06-18T03:07:40.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:40.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:07:40.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:07:40.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:40 smithi186 ceph-mon[67751]: Upgrade: Updating mon.c 2024-06-18T03:07:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:07:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:07:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:41.327 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:41 smithi163 systemd[1]: Stopping Ceph mon.c for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:07:41.710 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:41 smithi163 ceph-mon[82879]: Deploying daemon mon.c on smithi163 2024-06-18T03:07:41.710 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:41 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:41.710 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:41 smithi163 ceph-mon[82879]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T03:07:41.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:41 smithi163 ceph-mon[29791]: Deploying daemon mon.c on smithi163 2024-06-18T03:07:41.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:41 smithi163 ceph-mon[29791]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:41.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:41 smithi163 ceph-mon[29791]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T03:07:41.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:41 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c[29787]: 2024-06-18T03:07:41.333+0000 7f58b2cf4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:07:41.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:41 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c[29787]: 2024-06-18T03:07:41.333+0000 7f58b2cf4700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-06-18T03:07:41.710 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:41 smithi163 podman[87533]: 2024-06-18 03:07:41.405167846 +0000 UTC m=+0.199642807 container died 158231a62b0cd5f04510d08350f49396ac13d07825721c2ca4332eb9603cf6ba (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.component=centos-stream-container, ceph=True, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, release=754, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public) 2024-06-18T03:07:41.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:41 smithi186 ceph-mon[67751]: Deploying daemon mon.c on smithi163 2024-06-18T03:07:41.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:41 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:41.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:41 smithi186 ceph-mon[67751]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T03:07:42.026 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:41 smithi163 podman[87533]: 2024-06-18 03:07:41.817047691 +0000 UTC m=+0.611522647 container cleanup 158231a62b0cd5f04510d08350f49396ac13d07825721c2ca4332eb9603cf6ba (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c, RELEASE=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, ceph=True, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, release=754, architecture=x86_64, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0) 2024-06-18T03:07:42.026 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:41 smithi163 bash[87533]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c 2024-06-18T03:07:42.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 podman[87545]: 2024-06-18 03:07:42.026859776 +0000 UTC m=+0.619262518 container remove 158231a62b0cd5f04510d08350f49396ac13d07825721c2ca4332eb9603cf6ba (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , vcs-type=git, RELEASE=HEAD, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, version=8, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, distribution-scope=public) 2024-06-18T03:07:42.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.c.service: Deactivated successfully. 2024-06-18T03:07:42.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 systemd[1]: Stopped Ceph mon.c for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:07:42.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.c.service: Consumed 15.162s CPU time. 2024-06-18T03:07:42.697 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 systemd[1]: Starting Ceph mon.c for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:07:42.697 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 podman[87677]: 2024-06-18 03:07:42.48649002 +0000 UTC m=+0.017052606 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:07:42.697 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 podman[87677]: 2024-06-18 03:07:42.594228984 +0000 UTC m=+0.124791570 container create 9950de11350dd7ede23a5bc68fe602e24120cab1c49148a1b541bfda46aae0b2 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c, GIT_BRANCH=HEAD, RELEASE=squid-3781e3a, org.label-schema.build-date=20240612, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, ceph=True, org.label-schema.schema-version=1.0) 2024-06-18T03:07:43.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 podman[87677]: 2024-06-18 03:07:42.797070823 +0000 UTC m=+0.327633418 container init 9950de11350dd7ede23a5bc68fe602e24120cab1c49148a1b541bfda46aae0b2 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240612) 2024-06-18T03:07:43.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 podman[87677]: 2024-06-18 03:07:42.800439016 +0000 UTC m=+0.331001602 container start 9950de11350dd7ede23a5bc68fe602e24120cab1c49148a1b541bfda46aae0b2 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS) 2024-06-18T03:07:43.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: set uid:gid to 167:167 (ceph:ceph) 2024-06-18T03:07:43.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc), process ceph-mon, pid 2 2024-06-18T03:07:43.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: pidfile_write: ignore empty --pid-file 2024-06-18T03:07:43.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: load: jerasure load: lrc 2024-06-18T03:07:43.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: RocksDB version: 7.9.2 2024-06-18T03:07:43.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Git sha 0 2024-06-18T03:07:43.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Compile date 2024-06-17 15:21:02 2024-06-18T03:07:43.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: DB SUMMARY 2024-06-18T03:07:43.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: DB Session ID: C4LMFZAXNCH8S8KR9PYD 2024-06-18T03:07:43.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: CURRENT file: CURRENT 2024-06-18T03:07:43.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: IDENTITY file: IDENTITY 2024-06-18T03:07:43.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3198 Bytes 2024-06-18T03:07:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000060.sst 2024-06-18T03:07:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000058.log size: 2059017 ; 2024-06-18T03:07:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.error_if_exists: 0 2024-06-18T03:07:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.create_if_missing: 0 2024-06-18T03:07:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.paranoid_checks: 1 2024-06-18T03:07:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.flush_verify_memtable_count: 1 2024-06-18T03:07:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-18T03:07:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-06-18T03:07:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.env: 0x55d6242e2740 2024-06-18T03:07:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.fs: PosixFileSystem 2024-06-18T03:07:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.info_log: 0x55d62563f000 2024-06-18T03:07:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_file_opening_threads: 16 2024-06-18T03:07:43.299 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.statistics: (nil) 2024-06-18T03:07:43.300 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.use_fsync: 0 2024-06-18T03:07:43.300 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_log_file_size: 0 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.keep_log_file_num: 1000 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.recycle_log_file_num: 0 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.allow_fallocate: 1 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.allow_mmap_reads: 0 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.allow_mmap_writes: 0 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.use_direct_reads: 0 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.create_missing_column_families: 0 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.db_log_dir: 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.wal_dir: 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-18T03:07:43.301 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.advise_random_on_open: 1 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.db_write_buffer_size: 0 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.write_buffer_manager: 0x55d62564ec80 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.rate_limiter: (nil) 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.wal_recovery_mode: 2 2024-06-18T03:07:43.302 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.enable_thread_tracking: 0 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.enable_pipelined_write: 0 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.unordered_write: 0 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.row_cache: None 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.wal_filter: None 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.allow_ingest_behind: 0 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.two_write_queues: 0 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.manual_wal_flush: 0 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.wal_compression: 0 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.atomic_flush: 0 2024-06-18T03:07:43.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-18T03:07:43.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-18T03:07:43.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-18T03:07:43.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.log_readahead_size: 0 2024-06-18T03:07:43.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-18T03:07:43.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.best_efforts_recovery: 0 2024-06-18T03:07:43.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-18T03:07:43.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-18T03:07:43.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.allow_data_in_errors: 0 2024-06-18T03:07:43.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.db_host_id: __hostname__ 2024-06-18T03:07:43.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.enforce_single_del_contracts: true 2024-06-18T03:07:43.304 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_background_jobs: 2 2024-06-18T03:07:43.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_background_compactions: -1 2024-06-18T03:07:43.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_subcompactions: 1 2024-06-18T03:07:43.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-18T03:07:43.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-18T03:07:43.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-18T03:07:43.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_total_wal_size: 0 2024-06-18T03:07:43.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-18T03:07:43.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-18T03:07:43.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-18T03:07:43.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-18T03:07:43.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_open_files: -1 2024-06-18T03:07:43.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.bytes_per_sync: 0 2024-06-18T03:07:43.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-18T03:07:43.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-18T03:07:43.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compaction_readahead_size: 0 2024-06-18T03:07:43.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_background_flushes: -1 2024-06-18T03:07:43.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Compression algorithms supported: 2024-06-18T03:07:43.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: kZSTD supported: 0 2024-06-18T03:07:43.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: kXpressCompression supported: 0 2024-06-18T03:07:43.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: kBZip2Compression supported: 0 2024-06-18T03:07:43.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-18T03:07:43.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: kLZ4Compression supported: 1 2024-06-18T03:07:43.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: kZlibCompression supported: 1 2024-06-18T03:07:43.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: kLZ4HCCompression supported: 1 2024-06-18T03:07:43.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: kSnappyCompression supported: 1 2024-06-18T03:07:43.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: DMutex implementation: pthread_mutex_t 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.merge_operator: 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compaction_filter: None 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compaction_filter_factory: None 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.sst_partitioner_factory: None 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d62563ed00) 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout: cache_index_and_filter_blocks: 1 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-18T03:07:43.307 INFO:journalctl@ceph.mon.c.smithi163.stdout: pin_top_level_index_and_filter: 1 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: index_type: 0 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: data_block_index_type: 0 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: index_shortening: 1 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: checksum: 4 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: no_block_cache: 0 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_cache: 0x55d625635090 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_cache_name: BinnedLRUCache 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_cache_options: 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: capacity : 536870912 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: num_shard_bits : 4 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: strict_capacity_limit : 0 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: high_pri_pool_ratio: 0.000 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_cache_compressed: (nil) 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: persistent_cache: (nil) 2024-06-18T03:07:43.308 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_size: 4096 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_size_deviation: 10 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_restart_interval: 16 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: index_block_restart_interval: 1 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: metadata_block_size: 4096 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: partition_filters: 0 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: use_delta_encoding: 1 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: filter_policy: bloomfilter 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: whole_key_filtering: 1 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: verify_compression: 0 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: read_amp_bytes_per_bit: 0 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: format_version: 5 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: enable_index_compression: 1 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_align: 0 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: max_auto_readahead_size: 262144 2024-06-18T03:07:43.309 INFO:journalctl@ceph.mon.c.smithi163.stdout: prepopulate_block_cache: 0 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout: initial_auto_readahead_size: 8192 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout: num_file_reads_for_auto_readahead: 2 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.write_buffer_size: 33554432 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_write_buffer_number: 2 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compression: NoCompression 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.bottommost_compression: Disabled 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.prefix_extractor: nullptr 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.num_levels: 7 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-18T03:07:43.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-18T03:07:43.311 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-18T03:07:43.311 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-18T03:07:43.311 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-18T03:07:43.311 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-18T03:07:43.311 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-06-18T03:07:43.311 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-06-18T03:07:43.311 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-18T03:07:43.311 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compression_opts.level: 32767 2024-06-18T03:07:43.311 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compression_opts.strategy: 0 2024-06-18T03:07:43.311 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-18T03:07:43.311 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-18T03:07:43.311 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compression_opts.enabled: false 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.target_file_size_base: 67108864 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-18T03:07:43.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.arena_block_size: 1048576 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.disable_auto_compactions: 0 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-18T03:07:43.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.table_properties_collectors: 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.inplace_update_support: 0 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.bloom_locality: 0 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.max_successive_merges: 0 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.paranoid_file_checks: 0 2024-06-18T03:07:43.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.force_consistency_checks: 1 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.report_bg_io_stats: 0 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.ttl: 2592000 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.enable_blob_files: false 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.min_blob_size: 0 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.blob_file_size: 268435456 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.blob_file_starting_level: 0 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-06-18T03:07:43.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-06-18T03:07:43.316 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-18T03:07:43.316 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 62, last_sequence is 32000, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-18T03:07:43.316 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-06-18T03:07:43.316 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: a1a3b1bc-dcb0-4adb-97b3-3eda84f31454 2024-06-18T03:07:43.316 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718680062826032, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-06-18T03:07:43.316 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-06-18T03:07:43.316 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718680062831426, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 1291568, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 32005, "largest_seqno": 32378, "table_properties": {"data_size": 1288807, "index_size": 1299, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 453, "raw_key_size": 4280, "raw_average_key_size": 24, "raw_value_size": 1284756, "raw_average_value_size": 7258, "num_data_blocks": 58, "num_entries": 177, "num_filter_entries": 177, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1718680062, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "a1a3b1bc-dcb0-4adb-97b3-3eda84f31454", "db_session_id": "C4LMFZAXNCH8S8KR9PYD", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-06-18T03:07:43.316 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718680062831563, "job": 1, "event": "recovery_finished"} 2024-06-18T03:07:43.316 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-06-18T03:07:43.316 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d625662e00 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: DB pointer 0x55d625758000 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** DB Stats ** 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** Compaction Stats [default] ** 2024-06-18T03:07:43.317 INFO:journalctl@ceph.mon.c.smithi163.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: L0 1/0 1.23 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 483.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: L6 1/0 10.22 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: Sum 2/0 11.45 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 483.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 483.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** Compaction Stats [default] ** 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 483.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2024-06-18T03:07:43.318 INFO:journalctl@ceph.mon.c.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout: Cumulative compaction: 0.00 GB write, 157.83 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout: Interval compaction: 0.00 GB write, 157.83 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout: Block cache BinnedLRUCache@0x55d625635090#2 capacity: 512.00 MB usage: 1.95 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.50 KB,9.53674e-05%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: starting mon.c rank 1 at public addrs [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] at bind addrs [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: mon.c@-1(???) e3 preinit fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T03:07:43.319 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: mon.c@-1(???).mds e1 new map 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: mon.c@-1(???).mds e1 print_map 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout: e1 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout: legacy client fscid: -1 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout: No filesystems configured 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: mon.c@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: mon.c@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: mon.c@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: mon.c@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 ceph-mon[87697]: mon.c@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 bash[87677]: 9950de11350dd7ede23a5bc68fe602e24120cab1c49148a1b541bfda46aae0b2 2024-06-18T03:07:43.320 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:42 smithi163 systemd[1]: Started Ceph mon.c for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:07:44.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:07:43] "GET /metrics HTTP/1.1" 200 37954 "" "Prometheus/2.43.0" 2024-06-18T03:07:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: 2: [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon.b 2024-06-18T03:07:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: fsmap 2024-06-18T03:07:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: osdmap e112: 8 total, 8 up, 8 in 2024-06-18T03:07:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: mgrmap e47: y(active, since 14s), standbys: x 2024-06-18T03:07:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:07:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:07:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in error state 2024-06-18T03:07:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: mon.b calling monitor election 2024-06-18T03:07:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: mon.a calling monitor election 2024-06-18T03:07:44.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: mon.c calling monitor election 2024-06-18T03:07:44.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:07:44.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:07:44.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:07:44.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-18T03:07:44.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: monmap epoch 4 2024-06-18T03:07:44.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T03:07:44.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: last_changed 2024-06-18T03:07:42.867396+0000 2024-06-18T03:07:44.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: created 2024-06-18T02:36:57.402480+0000 2024-06-18T03:07:44.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: min_mon_release 19 (squid) 2024-06-18T03:07:44.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: election_strategy: 1 2024-06-18T03:07:44.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: 0: [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] mon.a 2024-06-18T03:07:44.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: 1: [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] mon.c 2024-06-18T03:07:44.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: 2: [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon.b 2024-06-18T03:07:44.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: fsmap 2024-06-18T03:07:44.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: osdmap e112: 8 total, 8 up, 8 in 2024-06-18T03:07:44.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: mgrmap e47: y(active, since 14s), standbys: x 2024-06-18T03:07:44.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:07:44.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:07:44.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in error state 2024-06-18T03:07:44.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:44.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:44.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:07:44.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: 2: [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon.b 2024-06-18T03:07:44.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: fsmap 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: osdmap e112: 8 total, 8 up, 8 in 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: mgrmap e47: y(active, since 14s), standbys: x 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in error state 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: mon.b calling monitor election 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: mon.a calling monitor election 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: mon.c calling monitor election 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: monmap epoch 4 2024-06-18T03:07:44.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: last_changed 2024-06-18T03:07:42.867396+0000 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: created 2024-06-18T02:36:57.402480+0000 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: min_mon_release 19 (squid) 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: election_strategy: 1 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: 0: [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] mon.a 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: 1: [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] mon.c 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: 2: [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon.b 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: fsmap 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: osdmap e112: 8 total, 8 up, 8 in 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: mgrmap e47: y(active, since 14s), standbys: x 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in error state 2024-06-18T03:07:44.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:44.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:44.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:07:44.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: 2: [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon.b 2024-06-18T03:07:44.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: fsmap 2024-06-18T03:07:44.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: osdmap e112: 8 total, 8 up, 8 in 2024-06-18T03:07:44.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: mgrmap e47: y(active, since 14s), standbys: x 2024-06-18T03:07:44.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:07:44.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:07:44.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in error state 2024-06-18T03:07:44.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: mon.b calling monitor election 2024-06-18T03:07:44.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: mon.a calling monitor election 2024-06-18T03:07:44.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: mon.c calling monitor election 2024-06-18T03:07:44.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-18T03:07:44.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-18T03:07:44.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-18T03:07:44.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-18T03:07:44.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: monmap epoch 4 2024-06-18T03:07:44.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: fsid 929affee-2d1b-11ef-bc9d-c7b262605968 2024-06-18T03:07:44.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: last_changed 2024-06-18T03:07:42.867396+0000 2024-06-18T03:07:44.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: created 2024-06-18T02:36:57.402480+0000 2024-06-18T03:07:44.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: min_mon_release 19 (squid) 2024-06-18T03:07:44.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: election_strategy: 1 2024-06-18T03:07:44.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: 0: [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] mon.a 2024-06-18T03:07:44.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: 1: [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] mon.c 2024-06-18T03:07:44.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: 2: [v2:172.21.15.186:3300/0,v1:172.21.15.186:6789/0] mon.b 2024-06-18T03:07:44.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: fsmap 2024-06-18T03:07:44.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: osdmap e112: 8 total, 8 up, 8 in 2024-06-18T03:07:44.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: mgrmap e47: y(active, since 14s), standbys: x 2024-06-18T03:07:44.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:07:44.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:07:44.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in error state 2024-06-18T03:07:44.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:44.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:44.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:07:45.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:44 smithi163 ceph-mon[87697]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T03:07:45.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:44 smithi163 ceph-mon[82879]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T03:07:45.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:44 smithi186 ceph-mon[67751]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-18T03:07:47.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:47 smithi163 ceph-mon[87697]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T03:07:47.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:47 smithi163 ceph-mon[82879]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T03:07:47.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:47 smithi186 ceph-mon[67751]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T03:07:48.171 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:48 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:48.171 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:48 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:48.171 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:48 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:48.171 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:48 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:48.172 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:47 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:07:47.841+0000 7f10a413f640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-06-18T03:07:48.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:48 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:48.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:48 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:48.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:48 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:48.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:48 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:48.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:48 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:48.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:48 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:48.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:48 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:48.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:48 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:49.206 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:49 smithi163 ceph-mon[82879]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 873 B/s rd, 0 op/s 2024-06-18T03:07:49.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:49 smithi163 ceph-mon[87697]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 873 B/s rd, 0 op/s 2024-06-18T03:07:49.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:49 smithi186 ceph-mon[67751]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 873 B/s rd, 0 op/s 2024-06-18T03:07:50.541 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:07:51.229 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:07:51.230 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (2m) 4s ago 24m 28.8M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:07:51.230 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (2m) 18s ago 24m 76.6M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:07:51.230 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 4s ago 23m 58.9M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:07:51.230 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (3m) 18s ago 29m 438M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:07:51.230 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (13m) 4s ago 30m 488M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:07:51.230 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (29s) 4s ago 30m 40.6M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:07:51.230 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (80s) 18s ago 29m 41.4M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:07:51.230 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (8s) 4s ago 29m 22.4M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:07:51.230 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (3m) 4s ago 25m 15.7M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:07:51.230 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (3m) 18s ago 25m 15.7M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:07:51.231 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (28m) 4s ago 28m 56.4M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T03:07:51.231 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (28m) 4s ago 28m 56.8M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T03:07:51.231 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (27m) 4s ago 27m 51.4M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:07:51.231 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (27m) 4s ago 27m 55.0M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:07:51.231 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (27m) 18s ago 27m 61.6M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:07:51.231 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (26m) 18s ago 26m 52.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:07:51.231 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (26m) 18s ago 26m 52.2M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:07:51.231 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (26m) 18s ago 26m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:07:51.231 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (2m) 18s ago 25m 69.4M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:07:51.231 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (23m) 4s ago 23m 80.0M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:07:51.231 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (23m) 18s ago 23m 80.6M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:07:51.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:51.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[82879]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:51.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:51.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:51.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:51.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:07:51.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:51.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:07:51.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:07:51.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:51.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:51.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[87697]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:51.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:51.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:51.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:51.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:07:51.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:51.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:07:51.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:07:51.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:51 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:51.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:51 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:07:51.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:51 smithi186 ceph-mon[67751]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:51.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:51 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:51.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:51 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:51.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:51 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:51.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:51 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:07:51.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:51 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:51.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:51 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:07:51.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:51 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:07:51.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:51 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:51.637 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:07:51.637 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:07:51.637 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:07:51.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:07:51.637 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:07:51.637 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:07:51.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:07:51.637 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:07:51.637 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-18T03:07:51.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:07:51.638 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:07:51.638 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:07:51.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:07:51.638 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:07:51.638 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-06-18T03:07:51.638 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 5 2024-06-18T03:07:51.638 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:07:51.638 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:07:51.973 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:07:51.973 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:07:51.973 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:07:51.973 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi163", 2024-06-18T03:07:51.973 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [ 2024-06-18T03:07:51.973 INFO:teuthology.orchestra.run.smithi163.stdout: "mon" 2024-06-18T03:07:51.973 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2024-06-18T03:07:51.973 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "2/2 daemons upgraded", 2024-06-18T03:07:51.973 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading mon daemons", 2024-06-18T03:07:51.973 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:07:51.974 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:07:52.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[87697]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:52.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[87697]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 964 B/s rd, 0 op/s 2024-06-18T03:07:52.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[87697]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:52.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[87697]: Reconfiguring mon.a (monmap changed)... 2024-06-18T03:07:52.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[87697]: Reconfiguring daemon mon.a on smithi163 2024-06-18T03:07:52.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[87697]: from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:52.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[87697]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:52.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/1160629391' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:52.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:52.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:52.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T03:07:52.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T03:07:52.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:52.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[82879]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:52.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[82879]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 964 B/s rd, 0 op/s 2024-06-18T03:07:52.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[82879]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:52.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[82879]: Reconfiguring mon.a (monmap changed)... 2024-06-18T03:07:52.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[82879]: Reconfiguring daemon mon.a on smithi163 2024-06-18T03:07:52.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[82879]: from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:52.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[82879]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:52.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/1160629391' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:52.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:52.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:52.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T03:07:52.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T03:07:52.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:52 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:52.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:52 smithi186 ceph-mon[67751]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:52.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:52 smithi186 ceph-mon[67751]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 964 B/s rd, 0 op/s 2024-06-18T03:07:52.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:52 smithi186 ceph-mon[67751]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:52.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:52 smithi186 ceph-mon[67751]: Reconfiguring mon.a (monmap changed)... 2024-06-18T03:07:52.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:52 smithi186 ceph-mon[67751]: Reconfiguring daemon mon.a on smithi163 2024-06-18T03:07:52.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:52 smithi186 ceph-mon[67751]: from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:52.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:52 smithi186 ceph-mon[67751]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:52.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:52 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/1160629391' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:07:52.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:52 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:52.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:52 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:52.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:52 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T03:07:52.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:52 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T03:07:52.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:52 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:53.424 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:53 smithi163 ceph-mon[87697]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:53.424 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:53 smithi163 ceph-mon[87697]: Reconfiguring mgr.y (monmap changed)... 2024-06-18T03:07:53.424 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:53 smithi163 ceph-mon[87697]: Reconfiguring daemon mgr.y on smithi163 2024-06-18T03:07:53.424 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:53 smithi163 ceph-mon[82879]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:53.424 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:53 smithi163 ceph-mon[82879]: Reconfiguring mgr.y (monmap changed)... 2024-06-18T03:07:53.424 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:53 smithi163 ceph-mon[82879]: Reconfiguring daemon mgr.y on smithi163 2024-06-18T03:07:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:53 smithi186 ceph-mon[67751]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:07:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:53 smithi186 ceph-mon[67751]: Reconfiguring mgr.y (monmap changed)... 2024-06-18T03:07:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:53 smithi186 ceph-mon[67751]: Reconfiguring daemon mgr.y on smithi163 2024-06-18T03:07:54.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:07:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:07:53] "GET /metrics HTTP/1.1" 200 37954 "" "Prometheus/2.43.0" 2024-06-18T03:07:54.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:54 smithi163 ceph-mon[87697]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:54.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:54 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:54.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:54 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:54.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:54 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:07:54.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:54 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:07:54.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:54 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:54.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:54 smithi163 ceph-mon[82879]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:54.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:54 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:54.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:54 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:54.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:54 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:07:54.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:54 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:07:54.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:54 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:54.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:54 smithi186 ceph-mon[67751]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:54.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:54 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:54.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:54 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:54.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:54 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:07:54.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:54 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:07:54.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:54 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:55.446 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:55 smithi163 ceph-mon[87697]: Reconfiguring mon.c (monmap changed)... 2024-06-18T03:07:55.446 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:55 smithi163 ceph-mon[87697]: Reconfiguring daemon mon.c on smithi163 2024-06-18T03:07:55.446 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:55 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:55.446 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:55 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:55.446 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:55 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-18T03:07:55.446 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:55 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:55.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:55 smithi163 ceph-mon[82879]: Reconfiguring mon.c (monmap changed)... 2024-06-18T03:07:55.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:55 smithi163 ceph-mon[82879]: Reconfiguring daemon mon.c on smithi163 2024-06-18T03:07:55.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:55 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:55.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:55 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:55.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:55 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-18T03:07:55.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:55 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:55.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:55 smithi186 ceph-mon[67751]: Reconfiguring mon.c (monmap changed)... 2024-06-18T03:07:55.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:55 smithi186 ceph-mon[67751]: Reconfiguring daemon mon.c on smithi163 2024-06-18T03:07:55.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:55 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:55.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:55 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:55.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:55 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-18T03:07:55.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:55 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:56.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:56 smithi163 ceph-mon[87697]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:56.400 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:56 smithi163 ceph-mon[87697]: Reconfiguring osd.0 (monmap changed)... 2024-06-18T03:07:56.400 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:56 smithi163 ceph-mon[87697]: Reconfiguring daemon osd.0 on smithi163 2024-06-18T03:07:56.400 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:56 smithi163 ceph-mon[82879]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:56.400 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:56 smithi163 ceph-mon[82879]: Reconfiguring osd.0 (monmap changed)... 2024-06-18T03:07:56.400 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:56 smithi163 ceph-mon[82879]: Reconfiguring daemon osd.0 on smithi163 2024-06-18T03:07:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:56 smithi186 ceph-mon[67751]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:56 smithi186 ceph-mon[67751]: Reconfiguring osd.0 (monmap changed)... 2024-06-18T03:07:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:56 smithi186 ceph-mon[67751]: Reconfiguring daemon osd.0 on smithi163 2024-06-18T03:07:57.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:57.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:57.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[87697]: Reconfiguring osd.1 (monmap changed)... 2024-06-18T03:07:57.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-18T03:07:57.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:57.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[87697]: Reconfiguring daemon osd.1 on smithi163 2024-06-18T03:07:57.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[87697]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:57.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:57.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:57.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[82879]: Reconfiguring osd.1 (monmap changed)... 2024-06-18T03:07:57.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-18T03:07:57.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:57.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[82879]: Reconfiguring daemon osd.1 on smithi163 2024-06-18T03:07:57.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:57 smithi163 ceph-mon[82879]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:57 smithi186 ceph-mon[67751]: Reconfiguring osd.1 (monmap changed)... 2024-06-18T03:07:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-18T03:07:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:57 smithi186 ceph-mon[67751]: Reconfiguring daemon osd.1 on smithi163 2024-06-18T03:07:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:57 smithi186 ceph-mon[67751]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:07:59.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:59.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:59.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[87697]: Reconfiguring osd.2 (monmap changed)... 2024-06-18T03:07:59.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-18T03:07:59.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:59.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[87697]: Reconfiguring daemon osd.2 on smithi163 2024-06-18T03:07:59.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:07:59.261 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:59.261 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:59.261 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[82879]: Reconfiguring osd.2 (monmap changed)... 2024-06-18T03:07:59.261 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-18T03:07:59.261 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:59.261 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[82879]: Reconfiguring daemon osd.2 on smithi163 2024-06-18T03:07:59.261 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:58 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:07:59.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:58 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:59.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:58 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:07:59.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:58 smithi186 ceph-mon[67751]: Reconfiguring osd.2 (monmap changed)... 2024-06-18T03:07:59.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:58 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-18T03:07:59.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:58 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:07:59.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:58 smithi186 ceph-mon[67751]: Reconfiguring daemon osd.2 on smithi163 2024-06-18T03:07:59.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:58 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:08:00.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:59 smithi163 ceph-mon[82879]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 594 B/s rd, 0 op/s 2024-06-18T03:08:00.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:00.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:00.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-18T03:08:00.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:07:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:00.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:59 smithi163 ceph-mon[87697]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 594 B/s rd, 0 op/s 2024-06-18T03:08:00.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:00.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:00.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-18T03:08:00.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:07:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:59 smithi186 ceph-mon[67751]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 594 B/s rd, 0 op/s 2024-06-18T03:08:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-18T03:08:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:07:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:01.191 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[87697]: Reconfiguring osd.3 (monmap changed)... 2024-06-18T03:08:01.192 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[87697]: Reconfiguring daemon osd.3 on smithi163 2024-06-18T03:08:01.192 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:01.192 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:01.192 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:01.192 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T03:08:01.192 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:01.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[82879]: Reconfiguring osd.3 (monmap changed)... 2024-06-18T03:08:01.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[82879]: Reconfiguring daemon osd.3 on smithi163 2024-06-18T03:08:01.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:01.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:01.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:01.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T03:08:01.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:00 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:01.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:00 smithi186 ceph-mon[67751]: Reconfiguring osd.3 (monmap changed)... 2024-06-18T03:08:01.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:00 smithi186 ceph-mon[67751]: Reconfiguring daemon osd.3 on smithi163 2024-06-18T03:08:01.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:00 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:01.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:00 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:01.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:00 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:01.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:00 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T03:08:01.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:00 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:02.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:01 smithi163 ceph-mon[87697]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:08:02.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:01 smithi163 ceph-mon[87697]: Reconfiguring rgw.foo.smithi163.mvqwty (monmap changed)... 2024-06-18T03:08:02.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:01 smithi163 ceph-mon[87697]: Reconfiguring daemon rgw.foo.smithi163.mvqwty on smithi163 2024-06-18T03:08:02.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:01 smithi163 ceph-mon[82879]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:08:02.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:01 smithi163 ceph-mon[82879]: Reconfiguring rgw.foo.smithi163.mvqwty (monmap changed)... 2024-06-18T03:08:02.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:01 smithi163 ceph-mon[82879]: Reconfiguring daemon rgw.foo.smithi163.mvqwty on smithi163 2024-06-18T03:08:02.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:01 smithi186 ceph-mon[67751]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:08:02.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:01 smithi186 ceph-mon[67751]: Reconfiguring rgw.foo.smithi163.mvqwty (monmap changed)... 2024-06-18T03:08:02.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:01 smithi186 ceph-mon[67751]: Reconfiguring daemon rgw.foo.smithi163.mvqwty on smithi163 2024-06-18T03:08:03.516 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:03 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:03.516 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:03 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:03.516 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:03 smithi186 ceph-mon[67751]: Reconfiguring mon.b (monmap changed)... 2024-06-18T03:08:03.516 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:03 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:08:03.516 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:03 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:08:03.517 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:03 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:03.517 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:03 smithi186 ceph-mon[67751]: Reconfiguring daemon mon.b on smithi186 2024-06-18T03:08:03.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:03.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:03.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[87697]: Reconfiguring mon.b (monmap changed)... 2024-06-18T03:08:03.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:08:03.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:08:03.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:03.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[87697]: Reconfiguring daemon mon.b on smithi186 2024-06-18T03:08:03.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:03.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:03.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[82879]: Reconfiguring mon.b (monmap changed)... 2024-06-18T03:08:03.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-18T03:08:03.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-18T03:08:03.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:03.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-mon[82879]: Reconfiguring daemon mon.b on smithi186 2024-06-18T03:08:04.207 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:08:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:08:03] "GET /metrics HTTP/1.1" 200 38028 "" "Prometheus/2.43.0" 2024-06-18T03:08:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:04 smithi163 ceph-mon[82879]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T03:08:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T03:08:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:04.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:04 smithi163 ceph-mon[87697]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:04.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:04.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:04.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T03:08:04.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T03:08:04.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:04 smithi186 ceph-mon[67751]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-18T03:08:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-18T03:08:04.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:05.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:05 smithi186 ceph-mon[67751]: Reconfiguring mgr.x (monmap changed)... 2024-06-18T03:08:05.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:05 smithi186 ceph-mon[67751]: Reconfiguring daemon mgr.x on smithi186 2024-06-18T03:08:05.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:05.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:05.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-18T03:08:05.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:05.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:05 smithi163 ceph-mon[87697]: Reconfiguring mgr.x (monmap changed)... 2024-06-18T03:08:05.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:05 smithi163 ceph-mon[87697]: Reconfiguring daemon mgr.x on smithi186 2024-06-18T03:08:05.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:05.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:05.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-18T03:08:05.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:05.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:05 smithi163 ceph-mon[82879]: Reconfiguring mgr.x (monmap changed)... 2024-06-18T03:08:05.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:05 smithi163 ceph-mon[82879]: Reconfiguring daemon mgr.x on smithi186 2024-06-18T03:08:05.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:05.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:05.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-18T03:08:05.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:06.366 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:06 smithi186 ceph-mon[67751]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:06.366 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:06 smithi186 ceph-mon[67751]: Reconfiguring osd.4 (monmap changed)... 2024-06-18T03:08:06.366 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:06 smithi186 ceph-mon[67751]: Reconfiguring daemon osd.4 on smithi186 2024-06-18T03:08:06.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:06 smithi163 ceph-mon[87697]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:06 smithi163 ceph-mon[87697]: Reconfiguring osd.4 (monmap changed)... 2024-06-18T03:08:06.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:06 smithi163 ceph-mon[87697]: Reconfiguring daemon osd.4 on smithi186 2024-06-18T03:08:06.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:06 smithi163 ceph-mon[82879]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:06.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:06 smithi163 ceph-mon[82879]: Reconfiguring osd.4 (monmap changed)... 2024-06-18T03:08:06.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:06 smithi163 ceph-mon[82879]: Reconfiguring daemon osd.4 on smithi186 2024-06-18T03:08:07.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[87697]: Reconfiguring osd.5 (monmap changed)... 2024-06-18T03:08:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-18T03:08:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[87697]: Reconfiguring daemon osd.5 on smithi186 2024-06-18T03:08:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[87697]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:07.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:07.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:07.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[82879]: Reconfiguring osd.5 (monmap changed)... 2024-06-18T03:08:07.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-18T03:08:07.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:07.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[82879]: Reconfiguring daemon osd.5 on smithi186 2024-06-18T03:08:07.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:07 smithi163 ceph-mon[82879]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:07 smithi186 ceph-mon[67751]: Reconfiguring osd.5 (monmap changed)... 2024-06-18T03:08:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-18T03:08:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:07 smithi186 ceph-mon[67751]: Reconfiguring daemon osd.5 on smithi186 2024-06-18T03:08:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:07 smithi186 ceph-mon[67751]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:09.224 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:08 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:09.225 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:08 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:09.225 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:08 smithi186 ceph-mon[67751]: Reconfiguring osd.6 (monmap changed)... 2024-06-18T03:08:09.225 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:08 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-18T03:08:09.225 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:08 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:09.225 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:08 smithi186 ceph-mon[67751]: Reconfiguring daemon osd.6 on smithi186 2024-06-18T03:08:09.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:08 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:09.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:08 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:09.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:08 smithi163 ceph-mon[87697]: Reconfiguring osd.6 (monmap changed)... 2024-06-18T03:08:09.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:08 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-18T03:08:09.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:08 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:09.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:08 smithi163 ceph-mon[87697]: Reconfiguring daemon osd.6 on smithi186 2024-06-18T03:08:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:08 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:08 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:08 smithi163 ceph-mon[82879]: Reconfiguring osd.6 (monmap changed)... 2024-06-18T03:08:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:08 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-18T03:08:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:08 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:09.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:08 smithi163 ceph-mon[82879]: Reconfiguring daemon osd.6 on smithi186 2024-06-18T03:08:10.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[87697]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[87697]: Reconfiguring osd.7 (monmap changed)... 2024-06-18T03:08:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-18T03:08:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:10.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[87697]: Reconfiguring daemon osd.7 on smithi186 2024-06-18T03:08:10.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[82879]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:10.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:10.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:10.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[82879]: Reconfiguring osd.7 (monmap changed)... 2024-06-18T03:08:10.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-18T03:08:10.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:10.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:10 smithi163 ceph-mon[82879]: Reconfiguring daemon osd.7 on smithi186 2024-06-18T03:08:10.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:10 smithi186 ceph-mon[67751]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:10.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:10 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:10.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:10 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:10.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:10 smithi186 ceph-mon[67751]: Reconfiguring osd.7 (monmap changed)... 2024-06-18T03:08:10.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:10 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-18T03:08:10.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:10 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:10.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:10 smithi186 ceph-mon[67751]: Reconfiguring daemon osd.7 on smithi186 2024-06-18T03:08:11.282 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:11 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:11.282 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:11 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:11.282 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:11 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:11.282 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:11 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T03:08:11.282 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:11 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:11.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:11 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:11.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:11 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:11.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:11 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:11.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:11 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T03:08:11.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:11 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:11.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:11 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:11.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:11 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:11.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:11 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:11.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:11 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T03:08:11.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:11 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:12.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:12.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: Reconfiguring rgw.foo.smithi186.vxkxtx (monmap changed)... 2024-06-18T03:08:12.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: Reconfiguring daemon rgw.foo.smithi186.vxkxtx on smithi186 2024-06-18T03:08:12.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:08:12.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.321 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.322 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:08:12.323 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:12.324 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:08:12.325 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:12.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: Reconfiguring rgw.foo.smithi186.vxkxtx (monmap changed)... 2024-06-18T03:08:12.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: Reconfiguring daemon rgw.foo.smithi186.vxkxtx on smithi186 2024-06-18T03:08:12.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:12.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: Reconfiguring rgw.foo.smithi186.vxkxtx (monmap changed)... 2024-06-18T03:08:12.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: Reconfiguring daemon rgw.foo.smithi186.vxkxtx on smithi186 2024-06-18T03:08:12.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:08:12.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-06-18T03:08:12.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.548 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:08:12.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:08:12.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:12.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-18T03:08:12.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-18T03:08:12.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:08:12.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:08:12.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:08:12.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:08:12.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:08:12.552 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:08:12.552 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:08:12.552 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:08:12.552 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:08:12.552 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:08:12.552 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:08:12.552 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.552 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:08:12.552 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:08:12.552 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:08:12.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:08:12.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:08:12.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:08:12.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:08:12.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:08:12.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:08:12.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:08:12.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:08:12.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:08:12.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:08:12.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:08:12.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:08:12.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:08:12.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:08:12.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:08:12.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:08:12.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:08:12.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:08:12.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:08:12.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:08:12.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:08:12.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:08:12.556 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:12.556 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:08:12.556 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:12.556 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:12.556 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:08:12.556 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:13.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mon 2024-06-18T03:08:13.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all crash 2024-06-18T03:08:13.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mds 2024-06-18T03:08:13.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:08:13.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-18T03:08:13.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all nfs 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all node-exporter 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all prometheus 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all alertmanager 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all grafana 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all loki 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all promtail 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Finalizing container_image settings 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: Upgrade: Complete! 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 882 B/s rd, 0 op/s 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:08:13.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 663 B/s rd, 0 op/s 2024-06-18T03:08:13.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s 2024-06-18T03:08:13.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:13.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mon 2024-06-18T03:08:13.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all crash 2024-06-18T03:08:13.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mds 2024-06-18T03:08:13.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:08:13.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-18T03:08:13.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:08:13.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all nfs 2024-06-18T03:08:13.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:08:13.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all node-exporter 2024-06-18T03:08:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all prometheus 2024-06-18T03:08:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all alertmanager 2024-06-18T03:08:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all grafana 2024-06-18T03:08:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all loki 2024-06-18T03:08:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all promtail 2024-06-18T03:08:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Finalizing container_image settings 2024-06-18T03:08:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: Upgrade: Complete! 2024-06-18T03:08:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 882 B/s rd, 0 op/s 2024-06-18T03:08:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:08:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:08:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 663 B/s rd, 0 op/s 2024-06-18T03:08:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s 2024-06-18T03:08:13.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:13.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mon 2024-06-18T03:08:13.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all crash 2024-06-18T03:08:13.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mds 2024-06-18T03:08:13.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:08:13.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-18T03:08:13.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:08:13.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all nfs 2024-06-18T03:08:13.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:08:13.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all node-exporter 2024-06-18T03:08:13.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all prometheus 2024-06-18T03:08:13.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all alertmanager 2024-06-18T03:08:13.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all grafana 2024-06-18T03:08:13.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all loki 2024-06-18T03:08:13.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all promtail 2024-06-18T03:08:13.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Finalizing container_image settings 2024-06-18T03:08:13.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: Upgrade: Complete! 2024-06-18T03:08:13.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 882 B/s rd, 0 op/s 2024-06-18T03:08:13.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:08:13.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:13.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:08:13.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 663 B/s rd, 0 op/s 2024-06-18T03:08:13.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s 2024-06-18T03:08:13.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:14.225 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:08:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:08:13] "GET /metrics HTTP/1.1" 200 38025 "" "Prometheus/2.43.0" 2024-06-18T03:08:14.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:08:14.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:14.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:08:14.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:14.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:08:14.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:15.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:15 smithi163 ceph-mon[87697]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-06-18T03:08:15.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:15 smithi163 ceph-mon[82879]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-06-18T03:08:15.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:15 smithi186 ceph-mon[67751]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-06-18T03:08:16.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:16 smithi163 ceph-mon[87697]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s 2024-06-18T03:08:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:16 smithi163 ceph-mon[82879]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s 2024-06-18T03:08:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:16 smithi186 ceph-mon[67751]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s 2024-06-18T03:08:18.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:18 smithi163 ceph-mon[87697]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-06-18T03:08:18.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:18 smithi163 ceph-mon[82879]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-06-18T03:08:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:18 smithi186 ceph-mon[67751]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-06-18T03:08:20.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:20 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:20.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:20 smithi163 ceph-mon[87697]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:20 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:20 smithi163 ceph-mon[82879]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:20.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:20 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:20.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:20 smithi186 ceph-mon[67751]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:22.595 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:22 smithi163 ceph-mon[82879]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:08:22.595 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:22 smithi163 ceph-mon[82879]: from='client.54133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:22.595 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:22 smithi163 ceph-mon[87697]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:08:22.595 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:22 smithi163 ceph-mon[87697]: from='client.54133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:22.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:22 smithi186 ceph-mon[67751]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:08:22.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:22 smithi186 ceph-mon[67751]: from='client.54133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:23.063 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps' 2024-06-18T03:08:24.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:08:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:08:23] "GET /metrics HTTP/1.1" 200 38025 "" "Prometheus/2.43.0" 2024-06-18T03:08:24.603 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:24 smithi163 ceph-mon[87697]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:24.603 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:24 smithi163 ceph-mon[82879]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:24 smithi186 ceph-mon[67751]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:25.054 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:08:25.054 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (3m) 37s ago 24m 28.8M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:08:25.054 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (2m) 52s ago 24m 76.6M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:08:25.054 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 37s ago 24m 58.9M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:08:25.054 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (4m) 52s ago 29m 438M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:08:25.054 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (13m) 37s ago 31m 488M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:08:25.054 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (63s) 37s ago 31m 40.6M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:08:25.054 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (114s) 52s ago 29m 41.4M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:08:25.054 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (42s) 37s ago 30m 22.4M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:08:25.054 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (3m) 37s ago 25m 15.7M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:08:25.055 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (3m) 52s ago 25m 15.7M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:08:25.055 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (29m) 37s ago 29m 56.4M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T03:08:25.055 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (28m) 37s ago 28m 56.8M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T03:08:25.055 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (28m) 37s ago 28m 51.4M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:08:25.055 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (28m) 37s ago 28m 55.0M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:08:25.055 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (27m) 52s ago 27m 61.6M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:08:25.055 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (27m) 52s ago 27m 52.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:08:25.055 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (26m) 52s ago 26m 52.2M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:08:25.055 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (26m) 52s ago 26m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:08:25.055 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (3m) 52s ago 25m 69.4M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:08:25.055 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (24m) 37s ago 24m 80.0M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:08:25.055 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (24m) 52s ago 24m 80.6M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:08:25.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:25 smithi163 ceph-mon[87697]: from='client.54139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:25.693 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:25 smithi163 ceph-mon[82879]: from='client.54139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:25.716 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-06-18T03:08:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:25 smithi186 ceph-mon[67751]: from='client.54139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:26 smithi163 ceph-mon[82879]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:26 smithi163 ceph-mon[87697]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:26 smithi186 ceph-mon[67751]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:27.852 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:08:28.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:27 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/1367116812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:28.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:27 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/1367116812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:28.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:27 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/1367116812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:28.680 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-06-18T03:08:28.984 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:28 smithi163 ceph-mon[87697]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:28.984 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:28 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:08:28.985 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:28 smithi163 ceph-mon[82879]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:28.985 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:28 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:08:29.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:28 smithi186 ceph-mon[67751]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:29.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:28 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:08:30.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:30 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:30.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:30 smithi163 ceph-mon[82879]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:30.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:30 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:30.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:30 smithi163 ceph-mon[87697]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:30.646 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)" 2024-06-18T03:08:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:30 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:30 smithi186 ceph-mon[67751]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:31.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-06-18T03:08:31.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:31 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/726781243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:31.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:31 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/726781243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:31.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:31 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/726781243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:32.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:32 smithi163 ceph-mon[82879]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:32.649 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:32 smithi163 ceph-mon[87697]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:32 smithi186 ceph-mon[67751]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:34.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:08:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:08:33] "GET /metrics HTTP/1.1" 200 38023 "" "Prometheus/2.43.0" 2024-06-18T03:08:34.628 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:34 smithi163 ceph-mon[87697]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:34.629 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:34 smithi163 ceph-mon[87697]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:34.629 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:34 smithi163 ceph-mon[82879]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:34.629 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:34 smithi163 ceph-mon[82879]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:34.672 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:08:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:34 smithi186 ceph-mon[67751]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:34 smithi186 ceph-mon[67751]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:35.332 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade status' 2024-06-18T03:08:36.672 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:36 smithi163 ceph-mon[82879]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:36.672 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:36 smithi163 ceph-mon[87697]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:36 smithi186 ceph-mon[67751]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:37.393 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:08:37.393 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": null, 2024-06-18T03:08:37.393 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": false, 2024-06-18T03:08:37.393 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "", 2024-06-18T03:08:37.393 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:08:37.393 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": null, 2024-06-18T03:08:37.393 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "", 2024-06-18T03:08:37.394 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:08:37.394 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:08:37.994 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph health detail' 2024-06-18T03:08:38.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:38 smithi163 ceph-mon[87697]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:38.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:38 smithi163 ceph-mon[87697]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:38.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:38 smithi163 ceph-mon[82879]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:38.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:38 smithi163 ceph-mon[82879]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:38 smithi186 ceph-mon[67751]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:38 smithi186 ceph-mon[67751]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:40.146 INFO:teuthology.orchestra.run.smithi163.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:08:40.146 INFO:teuthology.orchestra.run.smithi163.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:08:40.147 INFO:teuthology.orchestra.run.smithi163.stdout: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in error state 2024-06-18T03:08:40.451 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:40 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/869912917' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:08:40.452 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:40 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/869912917' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:08:40.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:40 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/869912917' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:08:40.802 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-06-18T03:08:41.371 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:41 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:41.371 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:41 smithi163 ceph-mon[87697]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:41.371 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:41 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:41.372 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:41 smithi163 ceph-mon[82879]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:41.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:41 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:41.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:41 smithi186 ceph-mon[67751]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:42.537 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:42 smithi163 ceph-mon[87697]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:42.537 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:42 smithi163 ceph-mon[82879]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:42 smithi186 ceph-mon[67751]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:08:43.648 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:43 smithi163 ceph-mon[87697]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:43.648 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:08:43.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:43 smithi163 ceph-mon[82879]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:43.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:08:43.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:43 smithi186 ceph-mon[67751]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:43.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:08:44.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:08:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:08:43] "GET /metrics HTTP/1.1" 200 38021 "" "Prometheus/2.43.0" 2024-06-18T03:08:44.643 INFO:teuthology.orchestra.run.smithi163.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:08:44.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:44 smithi163 ceph-mon[82879]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:44.659 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:44 smithi163 ceph-mon[87697]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:44 smithi186 ceph-mon[67751]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:45.483 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-18T03:08:45.755 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[82879]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:08:45.755 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:45.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:45.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:08:45.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[82879]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 0 op/s 2024-06-18T03:08:45.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[82879]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:45.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:45.756 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[82879]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:08:45.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[87697]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:08:45.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:45.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:45.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:08:45.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[87697]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 0 op/s 2024-06-18T03:08:45.756 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[87697]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:45.757 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:45.757 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:45 smithi163 ceph-mon[87697]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:08:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:45 smithi186 ceph-mon[67751]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:08:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:45 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:45 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:45 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:08:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:45 smithi186 ceph-mon[67751]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 0 op/s 2024-06-18T03:08:46.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:45 smithi186 ceph-mon[67751]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:46.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:45 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:46.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:45 smithi186 ceph-mon[67751]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:08:47.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:47.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:08:47.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:08:47.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:08:47.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:47.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mgr 2024-06-18T03:08:47.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:47.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:47.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mon 2024-06-18T03:08:47.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:47.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:47.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all crash 2024-06-18T03:08:47.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:47.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-18T03:08:47.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-18T03:08:47.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: Upgrade: osd.0 is safe to restart 2024-06-18T03:08:47.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[87697]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:47.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:47.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:08:47.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:08:47.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:08:47.978 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:47.978 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mgr 2024-06-18T03:08:47.978 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:47.978 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:47.978 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mon 2024-06-18T03:08:47.978 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:47.978 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:47.978 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all crash 2024-06-18T03:08:47.978 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:47.978 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-18T03:08:47.978 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-18T03:08:47.978 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: Upgrade: osd.0 is safe to restart 2024-06-18T03:08:47.979 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:47 smithi163 ceph-mon[82879]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:48.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:48.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:08:48.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:08:48.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:08:48.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:48.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mgr 2024-06-18T03:08:48.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:48.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:48.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mon 2024-06-18T03:08:48.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:48.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:48.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all crash 2024-06-18T03:08:48.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:48.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-18T03:08:48.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-18T03:08:48.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: Upgrade: osd.0 is safe to restart 2024-06-18T03:08:48.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:47 smithi186 ceph-mon[67751]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:48.445 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:08:49.111 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:08:49.111 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (3m) 62s ago 25m 28.8M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:08:49.111 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (3m) 76s ago 25m 76.6M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:08:49.111 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 62s ago 24m 58.9M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:08:49.111 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (4m) 76s ago 29m 438M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:08:49.111 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (14m) 62s ago 31m 488M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:08:49.111 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (87s) 62s ago 31m 40.6M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:08:49.111 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (2m) 76s ago 30m 41.4M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (66s) 62s ago 30m 22.4M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (4m) 62s ago 26m 15.7M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (4m) 76s ago 26m 15.7M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (29m) 62s ago 29m 56.4M 2992M 17.2.0 e1d6a67b021e 6f9ccffb9dc3 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (29m) 62s ago 29m 56.8M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (28m) 62s ago 28m 51.4M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (28m) 62s ago 28m 55.0M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (28m) 76s ago 28m 61.6M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (27m) 76s ago 27m 52.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (27m) 76s ago 27m 52.2M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (26m) 76s ago 26m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (3m) 76s ago 26m 69.4M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (24m) 62s ago 24m 80.0M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:08:49.112 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (24m) 76s ago 24m 80.6M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:08:49.513 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:08:49.513 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:08:49.514 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:08:49.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:08:49.514 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:08:49.514 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:08:49.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:08:49.514 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:08:49.514 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-18T03:08:49.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:08:49.514 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:08:49.514 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:08:49.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:08:49.514 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:08:49.515 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-06-18T03:08:49.515 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 5 2024-06-18T03:08:49.515 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:08:49.515 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:08:49.870 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:08:49.870 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:08:49.870 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:08:49.870 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-06-18T03:08:49.870 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:08:49.871 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "0/8 daemons upgraded", 2024-06-18T03:08:49.871 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading osd daemons", 2024-06-18T03:08:49.871 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:08:49.871 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:08:50.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:50 smithi186 ceph-mon[67751]: from='client.44214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:50 smithi186 ceph-mon[67751]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:50 smithi186 ceph-mon[67751]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 590 B/s rd, 0 op/s 2024-06-18T03:08:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:50 smithi186 ceph-mon[67751]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:50 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:50 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-18T03:08:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:50 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:50.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:50 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/3008368432' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:50.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[82879]: from='client.44214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:50.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[82879]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:50.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[82879]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 590 B/s rd, 0 op/s 2024-06-18T03:08:50.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[82879]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:50.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:50.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-18T03:08:50.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:50.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/3008368432' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:50.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[87697]: from='client.44214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:50.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[87697]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:50.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[87697]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 590 B/s rd, 0 op/s 2024-06-18T03:08:50.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[87697]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:50.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:50.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-18T03:08:50.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:08:50.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:50 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/3008368432' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:08:51.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:51 smithi186 ceph-mon[67751]: Upgrade: Updating osd.0 2024-06-18T03:08:51.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:51 smithi186 ceph-mon[67751]: Deploying daemon osd.0 on smithi163 2024-06-18T03:08:51.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:51 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:51.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:51 smithi186 ceph-mon[67751]: from='client.54205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:51.371 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:51 smithi163 ceph-mon[82879]: Upgrade: Updating osd.0 2024-06-18T03:08:51.371 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:51 smithi163 ceph-mon[82879]: Deploying daemon osd.0 on smithi163 2024-06-18T03:08:51.371 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:51 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:51.371 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:51 smithi163 ceph-mon[82879]: from='client.54205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:51.371 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:51 smithi163 ceph-mon[87697]: Upgrade: Updating osd.0 2024-06-18T03:08:51.371 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:51 smithi163 ceph-mon[87697]: Deploying daemon osd.0 on smithi163 2024-06-18T03:08:51.371 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:51 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:08:51.372 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:51 smithi163 ceph-mon[87697]: from='client.54205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:08:52.088 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:52 smithi163 ceph-mon[87697]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:52.088 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:52 smithi163 ceph-mon[82879]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:52.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:52 smithi186 ceph-mon[67751]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:08:52.676 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:52 smithi163 systemd[1]: Stopping Ceph osd.0 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:08:53.042 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:52 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0[32629]: 2024-06-18T03:08:52.674+0000 7f6651b95700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:08:53.043 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:52 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0[32629]: 2024-06-18T03:08:52.674+0000 7f6651b95700 -1 osd.0 112 *** Got signal Terminated *** 2024-06-18T03:08:53.043 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:52 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0[32629]: 2024-06-18T03:08:52.674+0000 7f6651b95700 -1 osd.0 112 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:08:53.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:53 smithi186 ceph-mon[67751]: osd.0 marked itself down and dead 2024-06-18T03:08:53.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:53 smithi163 ceph-mon[87697]: osd.0 marked itself down and dead 2024-06-18T03:08:53.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:53 smithi163 ceph-mon[82879]: osd.0 marked itself down and dead 2024-06-18T03:08:53.843 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:53 smithi163 podman[94131]: 2024-06-18 03:08:53.554281329 +0000 UTC m=+0.983166956 container died 6f9ccffb9dc37476683d3e020a223f14293f455121a525ae42676f8ed052bfb7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_CLEAN=True, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, ceph=True, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, version=8, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-06-18T03:08:54.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:08:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:08:53] "GET /metrics HTTP/1.1" 200 38021 "" "Prometheus/2.43.0" 2024-06-18T03:08:54.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:54 smithi163 ceph-mon[82879]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 590 B/s rd, 0 op/s 2024-06-18T03:08:54.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:54 smithi163 ceph-mon[82879]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:08:54.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:54 smithi163 ceph-mon[82879]: osdmap e113: 8 total, 7 up, 8 in 2024-06-18T03:08:54.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:54 smithi163 ceph-mon[87697]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 590 B/s rd, 0 op/s 2024-06-18T03:08:54.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:54 smithi163 ceph-mon[87697]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:08:54.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:54 smithi163 ceph-mon[87697]: osdmap e113: 8 total, 7 up, 8 in 2024-06-18T03:08:54.294 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:54 smithi163 podman[94131]: 2024-06-18 03:08:54.009008643 +0000 UTC m=+1.437894269 container cleanup 6f9ccffb9dc37476683d3e020a223f14293f455121a525ae42676f8ed052bfb7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_CLEAN=True, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, name=centos-stream, vcs-type=git) 2024-06-18T03:08:54.294 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:54 smithi163 bash[94131]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0 2024-06-18T03:08:54.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:54 smithi186 ceph-mon[67751]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 590 B/s rd, 0 op/s 2024-06-18T03:08:54.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:54 smithi186 ceph-mon[67751]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:08:54.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:54 smithi186 ceph-mon[67751]: osdmap e113: 8 total, 7 up, 8 in 2024-06-18T03:08:54.693 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:54 smithi163 podman[94148]: 2024-06-18 03:08:54.329169195 +0000 UTC m=+0.775449952 container remove 6f9ccffb9dc37476683d3e020a223f14293f455121a525ae42676f8ed052bfb7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vendor=Red Hat, Inc., vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, ceph=True, release=754, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0) 2024-06-18T03:08:54.972 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:54 smithi163 podman[94234]: 2024-06-18 03:08:54.595961226 +0000 UTC m=+0.017246339 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:08:54.972 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:54 smithi163 podman[94234]: 2024-06-18 03:08:54.711411302 +0000 UTC m=+0.132696456 container create 83cac165708f119f4ff40ab24510321aeca8c09b3454df52e53e713e9ae47454 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240612, ceph=True, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-06-18T03:08:54.972 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:54 smithi163 podman[94234]: 2024-06-18 03:08:54.897652764 +0000 UTC m=+0.318937887 container init 83cac165708f119f4ff40ab24510321aeca8c09b3454df52e53e713e9ae47454 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:08:54.972 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:54 smithi163 podman[94234]: 2024-06-18 03:08:54.901164844 +0000 UTC m=+0.322449959 container start 83cac165708f119f4ff40ab24510321aeca8c09b3454df52e53e713e9ae47454 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:08:54.972 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:54 smithi163 podman[94234]: 2024-06-18 03:08:54.972154094 +0000 UTC m=+0.393439207 container attach 83cac165708f119f4ff40ab24510321aeca8c09b3454df52e53e713e9ae47454 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, org.label-schema.license=GPLv2) 2024-06-18T03:08:55.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:55 smithi163 ceph-mon[82879]: osdmap e114: 8 total, 7 up, 8 in 2024-06-18T03:08:55.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:55 smithi163 ceph-mon[87697]: osdmap e114: 8 total, 7 up, 8 in 2024-06-18T03:08:55.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:55 smithi163 podman[94234]: 2024-06-18 03:08:55.167965007 +0000 UTC m=+0.589250127 container died 83cac165708f119f4ff40ab24510321aeca8c09b3454df52e53e713e9ae47454 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-18T03:08:55.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:55 smithi186 ceph-mon[67751]: osdmap e114: 8 total, 7 up, 8 in 2024-06-18T03:08:56.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:56 smithi163 ceph-mon[87697]: pgmap v55: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:08:56.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:56 smithi163 ceph-mon[87697]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 21 pgs degraded (PG_DEGRADED) 2024-06-18T03:08:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:56 smithi163 ceph-mon[82879]: pgmap v55: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:08:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:56 smithi163 ceph-mon[82879]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 21 pgs degraded (PG_DEGRADED) 2024-06-18T03:08:56.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:55 smithi163 podman[94257]: 2024-06-18 03:08:55.851128218 +0000 UTC m=+0.677378414 container remove 83cac165708f119f4ff40ab24510321aeca8c09b3454df52e53e713e9ae47454 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-3781e3a, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0) 2024-06-18T03:08:56.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:55 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.0.service: Deactivated successfully. 2024-06-18T03:08:56.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:55 smithi163 systemd[1]: Stopped Ceph osd.0 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:08:56.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:55 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.0.service: Consumed 13.740s CPU time. 2024-06-18T03:08:56.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 systemd[1]: Starting Ceph osd.0 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:08:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:56 smithi186 ceph-mon[67751]: pgmap v55: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:08:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:56 smithi186 ceph-mon[67751]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 21 pgs degraded (PG_DEGRADED) 2024-06-18T03:08:56.587 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 podman[94354]: 2024-06-18 03:08:56.20451514 +0000 UTC m=+0.017246534 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:08:56.587 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 podman[94354]: 2024-06-18 03:08:56.336596548 +0000 UTC m=+0.149327932 container create 399f5d7bfdec3b8e236db94b254b7eaf3f1c7251e2985c943000c5225279235c (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-activate, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-06-18T03:08:56.963 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 podman[94354]: 2024-06-18 03:08:56.587684725 +0000 UTC m=+0.400416119 container init 399f5d7bfdec3b8e236db94b254b7eaf3f1c7251e2985c943000c5225279235c (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:08:56.964 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 podman[94354]: 2024-06-18 03:08:56.591286959 +0000 UTC m=+0.404018340 container start 399f5d7bfdec3b8e236db94b254b7eaf3f1c7251e2985c943000c5225279235c (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:08:56.964 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 podman[94354]: 2024-06-18 03:08:56.679465572 +0000 UTC m=+0.492196964 container attach 399f5d7bfdec3b8e236db94b254b7eaf3f1c7251e2985c943000c5225279235c (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-06-18T03:08:57.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-activate[94369]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-18T03:08:57.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 bash[94354]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-18T03:08:57.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-activate[94369]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-18T03:08:57.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 bash[94354]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-18T03:08:57.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-activate[94369]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-18T03:08:57.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 bash[94354]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-18T03:08:57.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-activate[94369]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-18T03:08:57.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 bash[94354]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-18T03:08:57.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-activate[94369]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-06-18T03:08:57.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 bash[94354]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-06-18T03:08:57.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-activate[94369]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-18T03:08:57.294 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 bash[94354]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-18T03:08:57.294 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-activate[94369]: --> ceph-volume raw activate successful for osd ID: 0 2024-06-18T03:08:57.294 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:56 smithi163 bash[94354]: --> ceph-volume raw activate successful for osd ID: 0 2024-06-18T03:08:57.294 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:57 smithi163 podman[94354]: 2024-06-18 03:08:57.018426488 +0000 UTC m=+0.831157871 container died 399f5d7bfdec3b8e236db94b254b7eaf3f1c7251e2985c943000c5225279235c (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-06-18T03:08:57.745 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:57 smithi163 podman[94499]: 2024-06-18 03:08:57.532978709 +0000 UTC m=+0.508953455 container remove 399f5d7bfdec3b8e236db94b254b7eaf3f1c7251e2985c943000c5225279235c (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-activate, RELEASE=squid-3781e3a, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0) 2024-06-18T03:08:58.035 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:57 smithi163 podman[94561]: 2024-06-18 03:08:57.648144211 +0000 UTC m=+0.017831159 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:08:58.035 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:57 smithi163 podman[94561]: 2024-06-18 03:08:57.792284739 +0000 UTC m=+0.161971674 container create 9442cf4db7e9d068c139a3c526ba85bb123b2379af854cfdc127badc13ac1f55 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-3781e3a, maintainer=Guillaume Abrioux ) 2024-06-18T03:08:58.036 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:58 smithi163 podman[94561]: 2024-06-18 03:08:58.03602311 +0000 UTC m=+0.405710058 container init 9442cf4db7e9d068c139a3c526ba85bb123b2379af854cfdc127badc13ac1f55 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True) 2024-06-18T03:08:58.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:58 smithi163 ceph-mon[82879]: pgmap v56: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:08:58.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:58 smithi163 ceph-mon[87697]: pgmap v56: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:08:58.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:58 smithi163 podman[94561]: 2024-06-18 03:08:58.039465296 +0000 UTC m=+0.409152232 container start 9442cf4db7e9d068c139a3c526ba85bb123b2379af854cfdc127badc13ac1f55 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-06-18T03:08:58.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:58 smithi163 ceph-osd[94581]: -- 172.21.15.163:0/3506722395 <== mon.1 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d110d19a40 con 0x55d111ad8000 2024-06-18T03:08:58.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:58 smithi163 bash[94561]: 9442cf4db7e9d068c139a3c526ba85bb123b2379af854cfdc127badc13ac1f55 2024-06-18T03:08:58.293 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:58 smithi163 systemd[1]: Started Ceph osd.0 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:08:58.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:58 smithi186 ceph-mon[67751]: pgmap v56: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:08:58.792 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:08:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0[94575]: 2024-06-18T03:08:58.595+0000 7f83f6018640 -1 Falling back to public interface 2024-06-18T03:08:59.425 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.425 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.425 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.425 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.425 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:08:59.425 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.425 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.425 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.425 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.426 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.426 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.426 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:08:59.426 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.426 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:08:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:08:59.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:08:59.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:08:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:00.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:00 smithi163 ceph-mon[82879]: pgmap v57: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:09:00.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:00 smithi163 ceph-mon[87697]: pgmap v57: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:09:00.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:00 smithi186 ceph-mon[67751]: pgmap v57: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:09:01.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:01 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:01.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:01 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:01.543 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:09:01 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0[94575]: 2024-06-18T03:09:01.307+0000 7f83f6018640 -1 osd.0 0 read_superblock omap replica is missing. 2024-06-18T03:09:01.543 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:09:01 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0[94575]: 2024-06-18T03:09:01.337+0000 7f83f6018640 -1 osd.0 112 log_to_monitors true 2024-06-18T03:09:01.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:01 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:02.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:02 smithi163 ceph-mon[87697]: pgmap v58: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:09:02.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:02 smithi163 ceph-mon[87697]: from='osd.0 [v2:172.21.15.163:6802/1264613703,v1:172.21.15.163:6803/1264613703]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-18T03:09:02.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:02 smithi163 ceph-mon[87697]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-18T03:09:02.543 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:09:02 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0[94575]: 2024-06-18T03:09:02.270+0000 7f83eddc3640 -1 osd.0 112 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T03:09:02.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:02 smithi163 ceph-mon[82879]: pgmap v58: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:09:02.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:02 smithi163 ceph-mon[82879]: from='osd.0 [v2:172.21.15.163:6802/1264613703,v1:172.21.15.163:6803/1264613703]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-18T03:09:02.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:02 smithi163 ceph-mon[82879]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-18T03:09:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:02 smithi186 ceph-mon[67751]: pgmap v58: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:09:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:02 smithi186 ceph-mon[67751]: from='osd.0 [v2:172.21.15.163:6802/1264613703,v1:172.21.15.163:6803/1264613703]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-18T03:09:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:02 smithi186 ceph-mon[67751]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-18T03:09:03.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:03 smithi163 ceph-mon[87697]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-18T03:09:03.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:03 smithi163 ceph-mon[87697]: osdmap e115: 8 total, 7 up, 8 in 2024-06-18T03:09:03.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:03 smithi163 ceph-mon[87697]: from='osd.0 [v2:172.21.15.163:6802/1264613703,v1:172.21.15.163:6803/1264613703]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:09:03.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:03 smithi163 ceph-mon[87697]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:09:03.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:03 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:03.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:03 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:03.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:03 smithi163 ceph-mon[82879]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-18T03:09:03.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:03 smithi163 ceph-mon[82879]: osdmap e115: 8 total, 7 up, 8 in 2024-06-18T03:09:03.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:03 smithi163 ceph-mon[82879]: from='osd.0 [v2:172.21.15.163:6802/1264613703,v1:172.21.15.163:6803/1264613703]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:09:03.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:03 smithi163 ceph-mon[82879]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:09:03.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:03 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:03.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:03 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:03 smithi186 ceph-mon[67751]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-18T03:09:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:03 smithi186 ceph-mon[67751]: osdmap e115: 8 total, 7 up, 8 in 2024-06-18T03:09:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:03 smithi186 ceph-mon[67751]: from='osd.0 [v2:172.21.15.163:6802/1264613703,v1:172.21.15.163:6803/1264613703]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:09:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:03 smithi186 ceph-mon[67751]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:09:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:03 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:03.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:03 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:04.056 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:09:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:09:03] "GET /metrics HTTP/1.1" 200 38156 "" "Prometheus/2.43.0" 2024-06-18T03:09:04.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[82879]: pgmap v60: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:09:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[82879]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:09:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[82879]: osd.0 [v2:172.21.15.163:6802/1264613703,v1:172.21.15.163:6803/1264613703] boot 2024-06-18T03:09:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[82879]: osdmap e116: 8 total, 8 up, 8 in 2024-06-18T03:09:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T03:09:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[82879]: osdmap e117: 8 total, 8 up, 8 in 2024-06-18T03:09:04.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[87697]: pgmap v60: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:09:04.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[87697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:09:04.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[87697]: osd.0 [v2:172.21.15.163:6802/1264613703,v1:172.21.15.163:6803/1264613703] boot 2024-06-18T03:09:04.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[87697]: osdmap e116: 8 total, 8 up, 8 in 2024-06-18T03:09:04.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T03:09:04.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:04.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:04.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:04 smithi163 ceph-mon[87697]: osdmap e117: 8 total, 8 up, 8 in 2024-06-18T03:09:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:04 smithi186 ceph-mon[67751]: pgmap v60: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-18T03:09:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:04 smithi186 ceph-mon[67751]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:09:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:04 smithi186 ceph-mon[67751]: osd.0 [v2:172.21.15.163:6802/1264613703,v1:172.21.15.163:6803/1264613703] boot 2024-06-18T03:09:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:04 smithi186 ceph-mon[67751]: osdmap e116: 8 total, 8 up, 8 in 2024-06-18T03:09:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-18T03:09:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:04 smithi186 ceph-mon[67751]: osdmap e117: 8 total, 8 up, 8 in 2024-06-18T03:09:05.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:05 smithi186 ceph-mon[67751]: Health check update: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded (PG_DEGRADED) 2024-06-18T03:09:05.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:05 smithi163 ceph-mon[87697]: Health check update: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded (PG_DEGRADED) 2024-06-18T03:09:05.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:05 smithi163 ceph-mon[82879]: Health check update: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded (PG_DEGRADED) 2024-06-18T03:09:06.366 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:06 smithi163 ceph-mon[87697]: pgmap v63: 161 pgs: 13 active+undersized, 8 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 33/723 objects degraded (4.564%) 2024-06-18T03:09:06.366 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:06 smithi163 ceph-mon[82879]: pgmap v63: 161 pgs: 13 active+undersized, 8 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 33/723 objects degraded (4.564%) 2024-06-18T03:09:06.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:06 smithi186 ceph-mon[67751]: pgmap v63: 161 pgs: 13 active+undersized, 8 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 33/723 objects degraded (4.564%) 2024-06-18T03:09:06.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:09:06 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:09:06.524+0000 7f10a413f640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-06-18T03:09:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:09:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: pgmap v64: 161 pgs: 13 active+undersized, 8 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s; 33/723 objects degraded (4.564%) 2024-06-18T03:09:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: pgmap v65: 161 pgs: 13 active+undersized, 8 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 33/723 objects degraded (4.564%) 2024-06-18T03:09:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:09:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:07.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:07.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-18T03:09:07.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-18T03:09:07.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[87697]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-06-18T03:09:07.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:07.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:07.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:07.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:09:07.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: pgmap v64: 161 pgs: 13 active+undersized, 8 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s; 33/723 objects degraded (4.564%) 2024-06-18T03:09:07.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: pgmap v65: 161 pgs: 13 active+undersized, 8 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 33/723 objects degraded (4.564%) 2024-06-18T03:09:07.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:07.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:09:07.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:07.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:07.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:07.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-18T03:09:07.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-18T03:09:07.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:07 smithi163 ceph-mon[82879]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-06-18T03:09:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:09:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: pgmap v64: 161 pgs: 13 active+undersized, 8 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s; 33/723 objects degraded (4.564%) 2024-06-18T03:09:07.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: pgmap v65: 161 pgs: 13 active+undersized, 8 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 33/723 objects degraded (4.564%) 2024-06-18T03:09:07.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:07.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:09:07.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:07.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:07.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:07.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-18T03:09:07.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-18T03:09:07.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:07 smithi186 ceph-mon[67751]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-06-18T03:09:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:08 smithi163 ceph-mon[87697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded) 2024-06-18T03:09:08.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:08 smithi163 ceph-mon[82879]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded) 2024-06-18T03:09:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:08 smithi186 ceph-mon[67751]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/723 objects degraded (4.564%), 8 pgs degraded) 2024-06-18T03:09:09.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:09 smithi163 ceph-mon[87697]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-06-18T03:09:09.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:09 smithi163 ceph-mon[82879]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-06-18T03:09:09.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:09 smithi186 ceph-mon[67751]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-06-18T03:09:10.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:10 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:10.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:10 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:10.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:10 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:11.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:11 smithi163 ceph-mon[87697]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:09:11.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:11 smithi163 ceph-mon[82879]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:09:11.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:11 smithi186 ceph-mon[67751]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:09:14.293 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:09:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:09:13] "GET /metrics HTTP/1.1" 200 38159 "" "Prometheus/2.43.0" 2024-06-18T03:09:14.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:14 smithi163 ceph-mon[82879]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:09:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:09:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:14 smithi163 ceph-mon[87697]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:09:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:09:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:14 smithi186 ceph-mon[67751]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:09:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:09:16.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:16 smithi163 ceph-mon[87697]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:09:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:16 smithi163 ceph-mon[82879]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:09:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:16 smithi186 ceph-mon[67751]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:09:18.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:18 smithi163 ceph-mon[87697]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:09:18.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:18 smithi163 ceph-mon[82879]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:09:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:18 smithi186 ceph-mon[67751]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:09:19.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:19 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:19.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:19 smithi163 ceph-mon[87697]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:19 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:19 smithi163 ceph-mon[82879]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:19.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:19 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:19.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:19 smithi186 ceph-mon[67751]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:20.228 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:09:20.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:20 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:20.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:20 smithi163 ceph-mon[87697]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:20.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:20 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:20.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:20 smithi163 ceph-mon[82879]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:20.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:20 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:20.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:20 smithi186 ceph-mon[67751]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:20.878 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:09:20.878 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (4m) 18s ago 25m 28.8M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:09:20.879 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (3m) 108s ago 25m 76.6M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:09:20.879 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 18s ago 25m 58.9M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:09:20.879 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (5m) 108s ago 30m 438M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:09:20.879 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (14m) 18s ago 32m 496M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:09:20.879 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (119s) 18s ago 32m 46.5M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:09:20.879 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (2m) 108s ago 30m 41.4M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:09:20.879 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (98s) 18s ago 30m 37.4M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:09:20.879 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (4m) 18s ago 26m 15.7M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:09:20.879 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (4m) 108s ago 26m 15.7M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:09:20.879 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (23s) 18s ago 30m 12.0M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:09:20.879 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (29m) 18s ago 29m 57.5M 2992M 17.2.0 e1d6a67b021e 5c9bc4d2c280 2024-06-18T03:09:20.879 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (29m) 18s ago 29m 52.0M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:09:20.880 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (29m) 18s ago 29m 54.6M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:09:20.880 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (28m) 108s ago 28m 61.6M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:09:20.880 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (28m) 108s ago 28m 52.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:09:20.880 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (27m) 108s ago 27m 52.2M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:09:20.880 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (27m) 108s ago 27m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:09:20.880 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (4m) 108s ago 26m 69.4M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:09:20.880 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (25m) 18s ago 25m 80.3M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:09:20.880 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (25m) 108s ago 25m 80.6M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:09:21.270 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:09:21.270 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:09:21.270 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 1 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 6 2024-06-18T03:09:21.271 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:09:21.272 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:09:21.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:21 smithi163 ceph-mon[87697]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:09:21.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:21 smithi163 ceph-mon[87697]: from='client.44256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:21.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:21 smithi163 ceph-mon[87697]: from='client.54229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:21.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:21 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/3964619790' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:21 smithi163 ceph-mon[82879]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:09:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:21 smithi163 ceph-mon[82879]: from='client.44256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:21 smithi163 ceph-mon[82879]: from='client.54229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:21 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/3964619790' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:21.609 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:09:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:09:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:09:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-06-18T03:09:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:09:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "1/8 daemons upgraded", 2024-06-18T03:09:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading osd daemons", 2024-06-18T03:09:21.609 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:09:21.609 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:09:21.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:21 smithi186 ceph-mon[67751]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:09:21.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:21 smithi186 ceph-mon[67751]: from='client.44256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:21.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:21 smithi186 ceph-mon[67751]: from='client.54229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:21.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:21 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/3964619790' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:22.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:22 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-18T03:09:22.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:22 smithi163 ceph-mon[87697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-18T03:09:22.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:22 smithi163 ceph-mon[87697]: Upgrade: osd.1 is safe to restart 2024-06-18T03:09:22.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:22 smithi163 ceph-mon[87697]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:22.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:22 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-18T03:09:22.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:22 smithi163 ceph-mon[82879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-18T03:09:22.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:22 smithi163 ceph-mon[82879]: Upgrade: osd.1 is safe to restart 2024-06-18T03:09:22.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:22 smithi163 ceph-mon[82879]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:22.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:22 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-18T03:09:22.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:22 smithi186 ceph-mon[67751]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-18T03:09:22.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:22 smithi186 ceph-mon[67751]: Upgrade: osd.1 is safe to restart 2024-06-18T03:09:22.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:22 smithi186 ceph-mon[67751]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:24.153 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:09:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:09:23] "GET /metrics HTTP/1.1" 200 38159 "" "Prometheus/2.43.0" 2024-06-18T03:09:24.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:23 smithi163 ceph-mon[82879]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:24.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:23 smithi163 ceph-mon[82879]: Upgrade: Updating osd.1 2024-06-18T03:09:24.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:23 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:24.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:23 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-18T03:09:24.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:23 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:24.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:23 smithi163 ceph-mon[82879]: Deploying daemon osd.1 on smithi163 2024-06-18T03:09:24.154 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:23 smithi163 ceph-mon[87697]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:24.154 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:23 smithi163 ceph-mon[87697]: Upgrade: Updating osd.1 2024-06-18T03:09:24.154 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:23 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:24.154 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:23 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-18T03:09:24.154 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:23 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:24.154 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:23 smithi163 ceph-mon[87697]: Deploying daemon osd.1 on smithi163 2024-06-18T03:09:24.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:23 smithi186 ceph-mon[67751]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:24.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:23 smithi186 ceph-mon[67751]: Upgrade: Updating osd.1 2024-06-18T03:09:24.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:23 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:24.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:23 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-18T03:09:24.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:23 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:24.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:23 smithi186 ceph-mon[67751]: Deploying daemon osd.1 on smithi163 2024-06-18T03:09:25.793 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:25 smithi163 systemd[1]: Stopping Ceph osd.1 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:09:26.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:25 smithi163 ceph-mon[87697]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:09:26.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:25 smithi163 ceph-mon[82879]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:09:26.293 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1[35569]: 2024-06-18T03:09:25.964+0000 7feaa04bd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:09:26.294 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1[35569]: 2024-06-18T03:09:25.964+0000 7feaa04bd700 -1 osd.1 117 *** Got signal Terminated *** 2024-06-18T03:09:26.294 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1[35569]: 2024-06-18T03:09:25.964+0000 7feaa04bd700 -1 osd.1 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:09:26.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:25 smithi186 ceph-mon[67751]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:09:27.195 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:26 smithi163 ceph-mon[87697]: osd.1 marked itself down and dead 2024-06-18T03:09:27.195 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:26 smithi163 ceph-mon[82879]: osd.1 marked itself down and dead 2024-06-18T03:09:27.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:26 smithi186 ceph-mon[67751]: osd.1 marked itself down and dead 2024-06-18T03:09:27.543 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:27 smithi163 podman[99272]: 2024-06-18 03:09:27.406194687 +0000 UTC m=+1.562504627 container died 5c9bc4d2c280286c069e658b0bcb56b0ebc2b7b0ce75a5bdfa432b6a86797cd6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.openshift.expose-services=, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, name=centos-stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, RELEASE=HEAD, architecture=x86_64, vendor=Red Hat, Inc., release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container) 2024-06-18T03:09:27.899 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:27 smithi163 podman[99272]: 2024-06-18 03:09:27.75545086 +0000 UTC m=+1.911760795 container cleanup 5c9bc4d2c280286c069e658b0bcb56b0ebc2b7b0ce75a5bdfa432b6a86797cd6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1, RELEASE=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, release=754, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vcs-type=git, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=) 2024-06-18T03:09:27.899 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:27 smithi163 bash[99272]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1 2024-06-18T03:09:28.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:27 smithi163 ceph-mon[87697]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:28.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:27 smithi163 ceph-mon[87697]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:09:28.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:27 smithi163 ceph-mon[87697]: osdmap e118: 8 total, 7 up, 8 in 2024-06-18T03:09:28.293 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:28 smithi163 podman[99290]: 2024-06-18 03:09:28.065358384 +0000 UTC m=+0.659857623 container remove 5c9bc4d2c280286c069e658b0bcb56b0ebc2b7b0ce75a5bdfa432b6a86797cd6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1, name=centos-stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, release=754, vcs-type=git, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8) 2024-06-18T03:09:28.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:27 smithi163 ceph-mon[82879]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:28.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:27 smithi163 ceph-mon[82879]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:09:28.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:27 smithi163 ceph-mon[82879]: osdmap e118: 8 total, 7 up, 8 in 2024-06-18T03:09:28.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:27 smithi186 ceph-mon[67751]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:28.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:27 smithi186 ceph-mon[67751]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:09:28.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:27 smithi186 ceph-mon[67751]: osdmap e118: 8 total, 7 up, 8 in 2024-06-18T03:09:28.768 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:28 smithi163 podman[99389]: 2024-06-18 03:09:28.361133459 +0000 UTC m=+0.023460356 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:09:28.768 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:28 smithi163 podman[99389]: 2024-06-18 03:09:28.491217029 +0000 UTC m=+0.153543924 container create df01dc8a08119f645173f8d1ffa36677134fab159998cb3ff145815e10d14a63 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-3781e3a, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:09:28.768 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:28 smithi163 podman[99389]: 2024-06-18 03:09:28.693469603 +0000 UTC m=+0.355796511 container init df01dc8a08119f645173f8d1ffa36677134fab159998cb3ff145815e10d14a63 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-deactivate, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-06-18T03:09:28.768 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:28 smithi163 podman[99389]: 2024-06-18 03:09:28.696754854 +0000 UTC m=+0.359081757 container start df01dc8a08119f645173f8d1ffa36677134fab159998cb3ff145815e10d14a63 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:09:28.768 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:28 smithi163 podman[99389]: 2024-06-18 03:09:28.768399373 +0000 UTC m=+0.430726278 container attach df01dc8a08119f645173f8d1ffa36677134fab159998cb3ff145815e10d14a63 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-deactivate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-06-18T03:09:29.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:28 smithi163 ceph-mon[82879]: osdmap e119: 8 total, 7 up, 8 in 2024-06-18T03:09:29.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:28 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:29.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:28 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:09:29.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:28 smithi163 ceph-mon[87697]: osdmap e119: 8 total, 7 up, 8 in 2024-06-18T03:09:29.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:28 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:29.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:28 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:09:29.044 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:28 smithi163 podman[99389]: 2024-06-18 03:09:28.958731253 +0000 UTC m=+0.621058150 container died df01dc8a08119f645173f8d1ffa36677134fab159998cb3ff145815e10d14a63 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-deactivate, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a) 2024-06-18T03:09:29.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:28 smithi186 ceph-mon[67751]: osdmap e119: 8 total, 7 up, 8 in 2024-06-18T03:09:29.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:28 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:29.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:28 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:09:30.023 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:29 smithi163 podman[99414]: 2024-06-18 03:09:29.597219403 +0000 UTC m=+0.632549414 container remove df01dc8a08119f645173f8d1ffa36677134fab159998cb3ff145815e10d14a63 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-deactivate, ceph=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux ) 2024-06-18T03:09:30.023 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:29 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.1.service: Deactivated successfully. 2024-06-18T03:09:30.023 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:29 smithi163 systemd[1]: Stopped Ceph osd.1 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:09:30.023 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:29 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.1.service: Consumed 14.754s CPU time. 2024-06-18T03:09:30.023 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:29 smithi163 systemd[1]: Starting Ceph osd.1 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:09:30.275 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 podman[99511]: 2024-06-18 03:09:30.023153617 +0000 UTC m=+0.105539790 container create 84f332ab27881491c0c66cd4c2904d298860b16699e19cd7798544483f1b9a4b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.build-date=20240612) 2024-06-18T03:09:30.275 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 podman[99511]: 2024-06-18 03:09:29.935109341 +0000 UTC m=+0.017495484 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:09:30.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:30 smithi163 ceph-mon[87697]: pgmap v78: 161 pgs: 7 peering, 20 stale+active+clean, 134 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-18T03:09:30.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:30 smithi163 ceph-mon[82879]: pgmap v78: 161 pgs: 7 peering, 20 stale+active+clean, 134 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-18T03:09:30.543 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 podman[99511]: 2024-06-18 03:09:30.275436133 +0000 UTC m=+0.357822269 container init 84f332ab27881491c0c66cd4c2904d298860b16699e19cd7798544483f1b9a4b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-06-18T03:09:30.543 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 podman[99511]: 2024-06-18 03:09:30.278949721 +0000 UTC m=+0.361335852 container start 84f332ab27881491c0c66cd4c2904d298860b16699e19cd7798544483f1b9a4b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-activate, org.label-schema.build-date=20240612, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-3781e3a) 2024-06-18T03:09:30.543 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 podman[99511]: 2024-06-18 03:09:30.350401149 +0000 UTC m=+0.432787289 container attach 84f332ab27881491c0c66cd4c2904d298860b16699e19cd7798544483f1b9a4b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0) 2024-06-18T03:09:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:30 smithi186 ceph-mon[67751]: pgmap v78: 161 pgs: 7 peering, 20 stale+active+clean, 134 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-18T03:09:31.026 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-activate[99526]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-18T03:09:31.026 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 bash[99511]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-18T03:09:31.027 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-activate[99526]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-18T03:09:31.027 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 bash[99511]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-18T03:09:31.027 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-activate[99526]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-18T03:09:31.027 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 bash[99511]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-18T03:09:31.027 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-activate[99526]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-18T03:09:31.027 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 bash[99511]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-18T03:09:31.027 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-activate[99526]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-06-18T03:09:31.027 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 bash[99511]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-06-18T03:09:31.027 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-activate[99526]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-18T03:09:31.027 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 bash[99511]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-18T03:09:31.027 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-activate[99526]: --> ceph-volume raw activate successful for osd ID: 1 2024-06-18T03:09:31.027 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 bash[99511]: --> ceph-volume raw activate successful for osd ID: 1 2024-06-18T03:09:31.027 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:30 smithi163 podman[99511]: 2024-06-18 03:09:30.695055108 +0000 UTC m=+0.777441239 container died 84f332ab27881491c0c66cd4c2904d298860b16699e19cd7798544483f1b9a4b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-06-18T03:09:31.292 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:31 smithi163 podman[99640]: 2024-06-18 03:09:31.24560852 +0000 UTC m=+0.544698010 container remove 84f332ab27881491c0c66cd4c2904d298860b16699e19cd7798544483f1b9a4b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-activate, GIT_CLEAN=True, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, ceph=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:09:31.699 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:31 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:31.699 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:31 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:31.699 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:31 smithi163 podman[99697]: 2024-06-18 03:09:31.36482138 +0000 UTC m=+0.017506776 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:09:31.700 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:31 smithi163 podman[99697]: 2024-06-18 03:09:31.588240713 +0000 UTC m=+0.240926096 container create 63fcb171cec5509e202886a2c361abc65bb8c8b140d62466fe1513be2f1b806b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-18T03:09:31.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:31 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:32.043 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:31 smithi163 podman[99697]: 2024-06-18 03:09:31.806969217 +0000 UTC m=+0.459654615 container init 63fcb171cec5509e202886a2c361abc65bb8c8b140d62466fe1513be2f1b806b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240612, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-06-18T03:09:32.043 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:31 smithi163 podman[99697]: 2024-06-18 03:09:31.810212813 +0000 UTC m=+0.462898197 container start 63fcb171cec5509e202886a2c361abc65bb8c8b140d62466fe1513be2f1b806b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-06-18T03:09:32.043 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:31 smithi163 ceph-osd[99715]: -- 172.21.15.163:0/1165371341 <== mon.1 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a2f9523a40 con 0x55a2fa2e2000 2024-06-18T03:09:32.043 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:31 smithi163 bash[99697]: 63fcb171cec5509e202886a2c361abc65bb8c8b140d62466fe1513be2f1b806b 2024-06-18T03:09:32.043 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:31 smithi163 systemd[1]: Started Ceph osd.1 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:09:32.793 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:32 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1[99711]: 2024-06-18T03:09:32.351+0000 7f00092ff640 -1 Falling back to public interface 2024-06-18T03:09:32.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:32 smithi163 ceph-mon[87697]: pgmap v79: 161 pgs: 33 active+undersized, 7 peering, 23 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-06-18T03:09:32.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:32 smithi163 ceph-mon[87697]: Health check failed: Degraded data redundancy: 97/723 objects degraded (13.416%), 23 pgs degraded (PG_DEGRADED) 2024-06-18T03:09:32.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:32 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:32.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:32 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:32.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:32 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:32.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:32 smithi163 ceph-mon[82879]: pgmap v79: 161 pgs: 33 active+undersized, 7 peering, 23 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-06-18T03:09:32.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:32 smithi163 ceph-mon[82879]: Health check failed: Degraded data redundancy: 97/723 objects degraded (13.416%), 23 pgs degraded (PG_DEGRADED) 2024-06-18T03:09:32.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:32 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:32.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:32 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:32.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:32 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:32 smithi186 ceph-mon[67751]: pgmap v79: 161 pgs: 33 active+undersized, 7 peering, 23 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-06-18T03:09:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:32 smithi186 ceph-mon[67751]: Health check failed: Degraded data redundancy: 97/723 objects degraded (13.416%), 23 pgs degraded (PG_DEGRADED) 2024-06-18T03:09:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:32 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:32 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:32.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:32 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:34.141 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:09:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:09:33] "GET /metrics HTTP/1.1" 200 38156 "" "Prometheus/2.43.0" 2024-06-18T03:09:34.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:34 smithi163 ceph-mon[87697]: pgmap v80: 161 pgs: 39 active+undersized, 24 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 103/723 objects degraded (14.246%) 2024-06-18T03:09:34.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:34 smithi163 ceph-mon[82879]: pgmap v80: 161 pgs: 39 active+undersized, 24 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 103/723 objects degraded (14.246%) 2024-06-18T03:09:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:34 smithi186 ceph-mon[67751]: pgmap v80: 161 pgs: 39 active+undersized, 24 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 103/723 objects degraded (14.246%) 2024-06-18T03:09:35.347 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:35 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1[99711]: 2024-06-18T03:09:35.067+0000 7f00092ff640 -1 osd.1 0 read_superblock omap replica is missing. 2024-06-18T03:09:35.347 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:35 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1[99711]: 2024-06-18T03:09:35.096+0000 7f00092ff640 -1 osd.1 117 log_to_monitors true 2024-06-18T03:09:35.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:35 smithi163 ceph-mon[87697]: from='osd.1 [v2:172.21.15.163:6810/2323474862,v1:172.21.15.163:6811/2323474862]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-18T03:09:35.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:35 smithi163 ceph-mon[87697]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-18T03:09:35.793 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:09:35 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1[99711]: 2024-06-18T03:09:35.390+0000 7f00010aa640 -1 osd.1 117 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T03:09:35.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:35 smithi163 ceph-mon[82879]: from='osd.1 [v2:172.21.15.163:6810/2323474862,v1:172.21.15.163:6811/2323474862]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-18T03:09:35.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:35 smithi163 ceph-mon[82879]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-18T03:09:35.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:35 smithi186 ceph-mon[67751]: from='osd.1 [v2:172.21.15.163:6810/2323474862,v1:172.21.15.163:6811/2323474862]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-18T03:09:35.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:35 smithi186 ceph-mon[67751]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-18T03:09:36.769 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[87697]: pgmap v81: 161 pgs: 39 active+undersized, 24 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 103/723 objects degraded (14.246%) 2024-06-18T03:09:36.769 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[87697]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-18T03:09:36.769 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[87697]: osdmap e120: 8 total, 7 up, 8 in 2024-06-18T03:09:36.770 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[87697]: from='osd.1 [v2:172.21.15.163:6810/2323474862,v1:172.21.15.163:6811/2323474862]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:09:36.770 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[87697]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:09:36.770 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:36.770 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:36.770 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:36.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[82879]: pgmap v81: 161 pgs: 39 active+undersized, 24 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 103/723 objects degraded (14.246%) 2024-06-18T03:09:36.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[82879]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-18T03:09:36.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[82879]: osdmap e120: 8 total, 7 up, 8 in 2024-06-18T03:09:36.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[82879]: from='osd.1 [v2:172.21.15.163:6810/2323474862,v1:172.21.15.163:6811/2323474862]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:09:36.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[82879]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:09:36.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:36.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:36.771 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:36 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:36 smithi186 ceph-mon[67751]: pgmap v81: 161 pgs: 39 active+undersized, 24 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 103/723 objects degraded (14.246%) 2024-06-18T03:09:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:36 smithi186 ceph-mon[67751]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-18T03:09:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:36 smithi186 ceph-mon[67751]: osdmap e120: 8 total, 7 up, 8 in 2024-06-18T03:09:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:36 smithi186 ceph-mon[67751]: from='osd.1 [v2:172.21.15.163:6810/2323474862,v1:172.21.15.163:6811/2323474862]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:09:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:36 smithi186 ceph-mon[67751]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:09:36.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:36 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:36.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:36 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:36.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:36 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:37.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:37.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[82879]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:09:37.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[82879]: osd.1 [v2:172.21.15.163:6810/2323474862,v1:172.21.15.163:6811/2323474862] boot 2024-06-18T03:09:37.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[82879]: osdmap e121: 8 total, 8 up, 8 in 2024-06-18T03:09:37.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T03:09:37.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[82879]: pgmap v84: 161 pgs: 39 active+undersized, 24 active+undersized+degraded, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 103/723 objects degraded (14.246%) 2024-06-18T03:09:37.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[82879]: Health check update: Degraded data redundancy: 103/723 objects degraded (14.246%), 24 pgs degraded (PG_DEGRADED) 2024-06-18T03:09:37.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:37.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[87697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:09:37.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[87697]: osd.1 [v2:172.21.15.163:6810/2323474862,v1:172.21.15.163:6811/2323474862] boot 2024-06-18T03:09:37.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[87697]: osdmap e121: 8 total, 8 up, 8 in 2024-06-18T03:09:37.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T03:09:37.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[87697]: pgmap v84: 161 pgs: 39 active+undersized, 24 active+undersized+degraded, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 103/723 objects degraded (14.246%) 2024-06-18T03:09:37.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:37 smithi163 ceph-mon[87697]: Health check update: Degraded data redundancy: 103/723 objects degraded (14.246%), 24 pgs degraded (PG_DEGRADED) 2024-06-18T03:09:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:37 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:37.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:37 smithi186 ceph-mon[67751]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:09:37.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:37 smithi186 ceph-mon[67751]: osd.1 [v2:172.21.15.163:6810/2323474862,v1:172.21.15.163:6811/2323474862] boot 2024-06-18T03:09:37.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:37 smithi186 ceph-mon[67751]: osdmap e121: 8 total, 8 up, 8 in 2024-06-18T03:09:37.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:37 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-18T03:09:37.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:37 smithi186 ceph-mon[67751]: pgmap v84: 161 pgs: 39 active+undersized, 24 active+undersized+degraded, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 103/723 objects degraded (14.246%) 2024-06-18T03:09:37.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:37 smithi186 ceph-mon[67751]: Health check update: Degraded data redundancy: 103/723 objects degraded (14.246%), 24 pgs degraded (PG_DEGRADED) 2024-06-18T03:09:38.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:38 smithi186 ceph-mon[67751]: osdmap e122: 8 total, 8 up, 8 in 2024-06-18T03:09:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:38 smithi163 ceph-mon[82879]: osdmap e122: 8 total, 8 up, 8 in 2024-06-18T03:09:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:38 smithi163 ceph-mon[87697]: osdmap e122: 8 total, 8 up, 8 in 2024-06-18T03:09:39.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:39.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:39 smithi163 ceph-mon[82879]: pgmap v86: 161 pgs: 6 peering, 34 active+undersized, 23 active+undersized+degraded, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 97/723 objects degraded (13.416%) 2024-06-18T03:09:39.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:39.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:39 smithi163 ceph-mon[87697]: pgmap v86: 161 pgs: 6 peering, 34 active+undersized, 23 active+undersized+degraded, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 97/723 objects degraded (13.416%) 2024-06-18T03:09:39.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:39.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:39 smithi186 ceph-mon[67751]: pgmap v86: 161 pgs: 6 peering, 34 active+undersized, 23 active+undersized+degraded, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 97/723 objects degraded (13.416%) 2024-06-18T03:09:41.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:41.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:09:41.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: pgmap v87: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 975 B/s rd, 0 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: pgmap v88: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: Upgrade: Finalizing container_image settings 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:09:41.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:09:41.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:09:41.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:09:41.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:09:41.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:41.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:09:41.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: pgmap v87: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 975 B/s rd, 0 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: pgmap v88: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:09:41.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:09:41.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: Upgrade: Finalizing container_image settings 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:09:41.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: Upgrade: Complete! 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:09:41.048 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: pgmap v89: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: pgmap v90: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:09:41.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:09:41.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:09:41.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:09:41.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:09:41.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:09:41.049 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: Upgrade: Complete! 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: pgmap v89: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: pgmap v90: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:41.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:41.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:09:41.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: pgmap v91: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: pgmap v92: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.051 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.051 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:41.051 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.052 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:41.052 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:09:41.052 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: pgmap v91: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.052 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: pgmap v92: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.052 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:40 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:09:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: pgmap v87: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 975 B/s rd, 0 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: pgmap v88: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:09:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:09:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:41.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: Upgrade: Finalizing container_image settings 2024-06-18T03:09:41.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:09:41.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:09:41.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:09:41.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:09:41.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:09:41.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:09:41.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:09:41.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:09:41.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:09:41.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:09:41.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:09:41.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:09:41.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:09:41.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:09:41.071 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.072 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.072 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.072 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.072 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.072 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:09:41.072 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: Upgrade: Complete! 2024-06-18T03:09:41.072 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:09:41.072 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:09:41.072 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:41.072 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:09:41.072 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: pgmap v89: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.072 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: pgmap v90: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.073 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:41.073 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:41.073 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:09:41.073 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:09:41.073 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: pgmap v91: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.073 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: pgmap v92: 161 pgs: 6 peering, 15 active+undersized, 6 active+undersized+degraded, 134 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s; 37/723 objects degraded (5.118%) 2024-06-18T03:09:41.073 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:40 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:42.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:41 smithi163 ceph-mon[87697]: Health check update: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded (PG_DEGRADED) 2024-06-18T03:09:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:41 smithi163 ceph-mon[82879]: Health check update: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded (PG_DEGRADED) 2024-06-18T03:09:42.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:41 smithi186 ceph-mon[67751]: Health check update: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded (PG_DEGRADED) 2024-06-18T03:09:43.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:42 smithi163 ceph-mon[87697]: pgmap v93: 161 pgs: 6 peering, 155 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:09:43.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:42 smithi163 ceph-mon[82879]: pgmap v93: 161 pgs: 6 peering, 155 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:09:43.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:42 smithi186 ceph-mon[67751]: pgmap v93: 161 pgs: 6 peering, 155 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:09:44.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:09:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:09:43] "GET /metrics HTTP/1.1" 200 38178 "" "Prometheus/2.43.0" 2024-06-18T03:09:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:43 smithi163 ceph-mon[87697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded) 2024-06-18T03:09:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:09:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:44.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:43 smithi163 ceph-mon[82879]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded) 2024-06-18T03:09:44.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:44.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:09:44.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:44.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:43 smithi186 ceph-mon[67751]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded) 2024-06-18T03:09:44.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:44.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:09:44.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:45.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:44 smithi163 ceph-mon[87697]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:09:45.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:44 smithi163 ceph-mon[82879]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:09:45.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:44 smithi186 ceph-mon[67751]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:09:47.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:47 smithi163 ceph-mon[87697]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-06-18T03:09:47.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:47 smithi163 ceph-mon[82879]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-06-18T03:09:47.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:47 smithi186 ceph-mon[67751]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-06-18T03:09:49.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:49 smithi163 ceph-mon[87697]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-06-18T03:09:49.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:49 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:49.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:49 smithi163 ceph-mon[82879]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-06-18T03:09:49.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:49 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:49.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:49 smithi186 ceph-mon[67751]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-06-18T03:09:49.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:49 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:09:51.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:51 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:51.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:51 smithi163 ceph-mon[82879]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:09:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:51 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:51 smithi163 ceph-mon[87697]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:09:51.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:51 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:09:51.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:51 smithi186 ceph-mon[67751]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:09:52.649 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps' 2024-06-18T03:09:53.235 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:53 smithi163 ceph-mon[87697]: from='client.44286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:53.236 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:53 smithi163 ceph-mon[87697]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:53.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:53 smithi163 ceph-mon[82879]: from='client.44286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:53.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:53 smithi163 ceph-mon[82879]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:53.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:53 smithi186 ceph-mon[67751]: from='client.44286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:53.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:53 smithi186 ceph-mon[67751]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:54.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:09:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:09:53] "GET /metrics HTTP/1.1" 200 38178 "" "Prometheus/2.43.0" 2024-06-18T03:09:54.607 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:09:54.607 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (4m) 18s ago 26m 27.7M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:09:54.607 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (4m) 2m ago 26m 76.6M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:09:54.607 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 18s ago 25m 59.0M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:09:54.607 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (5m) 2m ago 31m 438M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:09:54.608 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (15m) 18s ago 32m 498M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:09:54.608 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (2m) 18s ago 32m 50.1M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:09:54.608 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (3m) 2m ago 31m 41.4M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:09:54.608 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (2m) 18s ago 31m 41.5M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:09:54.608 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (5m) 18s ago 27m 15.7M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:09:54.608 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (5m) 2m ago 27m 15.7M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:09:54.609 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (56s) 18s ago 30m 42.5M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:09:54.609 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (23s) 18s ago 30m 11.0M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:09:54.609 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (30m) 18s ago 30m 53.0M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:09:54.609 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (29m) 18s ago 29m 56.1M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:09:54.609 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (29m) 2m ago 29m 61.6M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:09:54.609 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (28m) 2m ago 28m 52.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:09:54.609 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (28m) 2m ago 28m 52.2M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:09:54.610 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (28m) 2m ago 28m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:09:54.610 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (4m) 2m ago 27m 69.4M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:09:54.610 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (25m) 18s ago 25m 80.4M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:09:54.610 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (25m) 2m ago 25m 80.6M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:09:55.228 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:55 smithi163 ceph-mon[87697]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:55.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:55 smithi163 ceph-mon[82879]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:55.252 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-06-18T03:09:55.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:55 smithi186 ceph-mon[67751]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:56.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:56 smithi163 ceph-mon[87697]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:56 smithi163 ceph-mon[82879]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:56.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:56 smithi186 ceph-mon[67751]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:09:57.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:57 smithi163 ceph-mon[87697]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:09:57.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:57 smithi163 ceph-mon[82879]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:09:57.298 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:09:57.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:57 smithi186 ceph-mon[67751]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:09:57.982 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-06-18T03:09:58.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:58 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/3423821289' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:58.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:58 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/3423821289' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:58.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:58 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/3423821289' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:09:59.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:59 smithi163 ceph-mon[87697]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:59.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:09:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:09:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:59 smithi163 ceph-mon[82879]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:59.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:09:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:09:59.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:59 smithi186 ceph-mon[67751]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:09:59.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:09:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:10:00.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:00 smithi186 ceph-mon[67751]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:10:00.440 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:00 smithi163 ceph-mon[87697]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:10:00.440 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:00 smithi163 ceph-mon[82879]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:10:01.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:01 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:01.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:01 smithi163 ceph-mon[82879]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:01.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:01 smithi163 ceph-mon[82879]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:01.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:01 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:01.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:01 smithi163 ceph-mon[87697]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:01.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:01 smithi163 ceph-mon[87697]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:01.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:01 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:01.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:01 smithi186 ceph-mon[67751]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:01.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:01 smithi186 ceph-mon[67751]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:01.374 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:10:02.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade status' 2024-06-18T03:10:03.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:02 smithi163 ceph-mon[87697]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:03.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:02 smithi163 ceph-mon[82879]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:03.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:02 smithi186 ceph-mon[67751]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:04.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:10:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:10:03] "GET /metrics HTTP/1.1" 200 38176 "" "Prometheus/2.43.0" 2024-06-18T03:10:04.079 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:10:04.079 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": null, 2024-06-18T03:10:04.079 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": false, 2024-06-18T03:10:04.079 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "", 2024-06-18T03:10:04.079 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:10:04.079 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": null, 2024-06-18T03:10:04.079 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "", 2024-06-18T03:10:04.079 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:10:04.079 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:10:04.883 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph health detail' 2024-06-18T03:10:05.188 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:05 smithi163 ceph-mon[87697]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:05.188 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:05 smithi163 ceph-mon[87697]: from='client.54277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:05.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:05 smithi163 ceph-mon[82879]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:05.189 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:05 smithi163 ceph-mon[82879]: from='client.54277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:05.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:05 smithi186 ceph-mon[67751]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:05.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:05 smithi186 ceph-mon[67751]: from='client.54277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:06.872 INFO:teuthology.orchestra.run.smithi163.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:10:06.873 INFO:teuthology.orchestra.run.smithi163.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:10:06.873 INFO:teuthology.orchestra.run.smithi163.stdout: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in error state 2024-06-18T03:10:07.228 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:07 smithi163 ceph-mon[87697]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:07.228 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:07 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/3735192540' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:10:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:07 smithi163 ceph-mon[82879]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:07 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/3735192540' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:10:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:07 smithi186 ceph-mon[67751]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:07 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/3735192540' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:10:07.519 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-06-18T03:10:09.144 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:09 smithi163 ceph-mon[87697]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:09.145 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:09 smithi163 ceph-mon[82879]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:09 smithi186 ceph-mon[67751]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:10.879 INFO:teuthology.orchestra.run.smithi163.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:10:11.252 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:11 smithi163 ceph-mon[87697]: from='client.44322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:11.253 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:11 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:11.253 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:11 smithi163 ceph-mon[87697]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:11.253 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:11 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:11.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:11 smithi163 ceph-mon[82879]: from='client.44322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:11.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:11 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:11.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:11 smithi163 ceph-mon[82879]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:11.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:11 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:11 smithi186 ceph-mon[67751]: from='client.44322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:11 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:11 smithi186 ceph-mon[67751]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:11 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:11.551 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-18T03:10:12.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:12 smithi163 ceph-mon[87697]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:10:12.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:12.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:12.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:12.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:10:12.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:12 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:12.526 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:12 smithi163 ceph-mon[82879]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:10:12.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:12.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:12.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:12.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:10:12.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:12 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:12.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:12 smithi186 ceph-mon[67751]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:10:12.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:12.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:12.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:12.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:10:12.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:12 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:13.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:13 smithi163 ceph-mon[87697]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-06-18T03:10:13.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:13 smithi163 ceph-mon[87697]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:13.474 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:13 smithi163 ceph-mon[87697]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:10:13.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:13 smithi163 ceph-mon[82879]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-06-18T03:10:13.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:13 smithi163 ceph-mon[82879]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:13.475 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:13 smithi163 ceph-mon[82879]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:10:13.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:13 smithi186 ceph-mon[67751]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-06-18T03:10:13.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:13 smithi186 ceph-mon[67751]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:13.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:13 smithi186 ceph-mon[67751]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:10:13.714 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:10:14.067 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:10:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:10:13] "GET /metrics HTTP/1.1" 200 38176 "" "Prometheus/2.43.0" 2024-06-18T03:10:14.371 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:10:14.372 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:10:14.377 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:10:14.377 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (4m) 38s ago 26m 27.7M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:10:14.377 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (4m) 2m ago 26m 76.6M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:10:14.377 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 38s ago 26m 59.0M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:10:14.377 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (6m) 2m ago 31m 438M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:10:14.378 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (15m) 38s ago 33m 498M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:10:14.378 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (2m) 38s ago 33m 50.1M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:10:14.378 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (3m) 2m ago 31m 41.4M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:10:14.378 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (2m) 38s ago 31m 41.5M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:10:14.378 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (5m) 38s ago 27m 15.7M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:10:14.378 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (5m) 2m ago 27m 15.7M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:10:14.378 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (76s) 38s ago 31m 42.5M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:10:14.378 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (43s) 38s ago 30m 11.0M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:10:14.378 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (30m) 38s ago 30m 53.0M 2992M 17.2.0 e1d6a67b021e 38b8f85b7dc0 2024-06-18T03:10:14.378 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (29m) 38s ago 29m 56.1M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:10:14.378 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (29m) 2m ago 29m 61.6M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:10:14.378 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (29m) 2m ago 29m 52.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:10:14.378 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (28m) 2m ago 28m 52.2M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:10:14.379 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (28m) 2m ago 28m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:10:14.379 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (5m) 2m ago 27m 69.4M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:10:14.379 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (26m) 38s ago 26m 80.4M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:10:14.379 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (26m) 2m ago 26m 80.6M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:10:14.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:10:14.781 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:10:14.781 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:10:14.781 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:10:14.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:10:14.781 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:10:14.781 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:10:14.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:10:14.781 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:10:14.781 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-06-18T03:10:14.781 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:10:14.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:10:14.782 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:10:14.782 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:10:14.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:10:14.782 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:10:14.782 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-06-18T03:10:14.782 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 7 2024-06-18T03:10:14.782 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:10:14.782 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:10:15.146 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:10:15.146 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:10:15.146 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:10:15.146 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-06-18T03:10:15.146 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:10:15.147 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "2/8 daemons upgraded", 2024-06-18T03:10:15.147 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading osd daemons", 2024-06-18T03:10:15.147 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:10:15.147 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:10:15.401 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[87697]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:15.401 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[87697]: from='client.54295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:15.401 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[87697]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:15.401 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:15.401 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/3718737319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:15.401 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:10:15.401 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:15.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:15.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:15.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:15.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:15.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:15.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-18T03:10:15.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[82879]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:15.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[82879]: from='client.54295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:15.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[82879]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:15.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:15.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/3718737319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:15.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:10:15.403 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:15.403 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:15.403 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:15.403 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:15.403 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:15.403 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:15.403 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:15 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-18T03:10:15.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:15 smithi186 ceph-mon[67751]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:15.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:15 smithi186 ceph-mon[67751]: from='client.54295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:15 smithi186 ceph-mon[67751]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:15 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:15 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/3718737319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:15 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:10:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:15 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:15 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:15 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:15 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:15 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:15 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:15 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-18T03:10:16.521 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[87697]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:16.521 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[87697]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:10:16.521 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[87697]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:10:16.521 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mgr 2024-06-18T03:10:16.522 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mon 2024-06-18T03:10:16.522 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all crash 2024-06-18T03:10:16.522 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[87697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-18T03:10:16.522 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[87697]: Upgrade: osd.2 is safe to restart 2024-06-18T03:10:16.522 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[87697]: from='client.44352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:16.522 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:16.522 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-18T03:10:16.522 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:16.522 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[82879]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:16.522 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[82879]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:10:16.522 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[82879]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:10:16.522 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mgr 2024-06-18T03:10:16.523 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mon 2024-06-18T03:10:16.523 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all crash 2024-06-18T03:10:16.523 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[82879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-18T03:10:16.523 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[82879]: Upgrade: osd.2 is safe to restart 2024-06-18T03:10:16.523 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[82879]: from='client.44352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:16.523 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:16.523 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-18T03:10:16.523 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:16 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:16.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:16 smithi186 ceph-mon[67751]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:16.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:16 smithi186 ceph-mon[67751]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:10:16.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:16 smithi186 ceph-mon[67751]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:10:16.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:16 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mgr 2024-06-18T03:10:16.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:16 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mon 2024-06-18T03:10:16.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:16 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all crash 2024-06-18T03:10:16.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:16 smithi186 ceph-mon[67751]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-18T03:10:16.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:16 smithi186 ceph-mon[67751]: Upgrade: osd.2 is safe to restart 2024-06-18T03:10:16.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:16 smithi186 ceph-mon[67751]: from='client.44352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:16.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:16 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:16.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:16 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-18T03:10:16.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:16 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:17.454 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:17 smithi163 ceph-mon[87697]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:17.454 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:17 smithi163 ceph-mon[87697]: Upgrade: Updating osd.2 2024-06-18T03:10:17.454 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:17 smithi163 ceph-mon[87697]: Deploying daemon osd.2 on smithi163 2024-06-18T03:10:17.455 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:17 smithi163 ceph-mon[82879]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:17.455 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:17 smithi163 ceph-mon[82879]: Upgrade: Updating osd.2 2024-06-18T03:10:17.455 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:17 smithi163 ceph-mon[82879]: Deploying daemon osd.2 on smithi163 2024-06-18T03:10:17.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:17 smithi186 ceph-mon[67751]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:17.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:17 smithi186 ceph-mon[67751]: Upgrade: Updating osd.2 2024-06-18T03:10:17.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:17 smithi186 ceph-mon[67751]: Deploying daemon osd.2 on smithi163 2024-06-18T03:10:19.284 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:19 smithi163 ceph-mon[87697]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:19.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:19 smithi163 ceph-mon[82879]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:19.285 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:19 smithi163 systemd[1]: Stopping Ceph osd.2 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:10:19.542 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2[38473]: 2024-06-18T03:10:19.282+0000 7f7f1645b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:10:19.543 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2[38473]: 2024-06-18T03:10:19.282+0000 7f7f1645b700 -1 osd.2 122 *** Got signal Terminated *** 2024-06-18T03:10:19.543 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2[38473]: 2024-06-18T03:10:19.283+0000 7f7f1645b700 -1 osd.2 122 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:10:19.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:19 smithi186 ceph-mon[67751]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:20.519 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:20 smithi163 ceph-mon[87697]: osd.2 marked itself down and dead 2024-06-18T03:10:20.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:20 smithi163 ceph-mon[82879]: osd.2 marked itself down and dead 2024-06-18T03:10:20.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:20 smithi186 ceph-mon[67751]: osd.2 marked itself down and dead 2024-06-18T03:10:20.792 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:20 smithi163 podman[105550]: 2024-06-18 03:10:20.5197371 +0000 UTC m=+1.347824491 container died 38b8f85b7dc05032f077d41f89aafed9831503352b332404a9605d317faa1583 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, version=8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, vcs-type=git, release=754, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-06-18T03:10:21.135 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:20 smithi163 podman[105550]: 2024-06-18 03:10:20.875711188 +0000 UTC m=+1.703798577 container cleanup 38b8f85b7dc05032f077d41f89aafed9831503352b332404a9605d317faa1583 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, architecture=x86_64, distribution-scope=public, maintainer=Guillaume Abrioux , name=centos-stream, release=754, RELEASE=HEAD, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, vcs-type=git) 2024-06-18T03:10:21.135 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:20 smithi163 bash[105550]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2 2024-06-18T03:10:21.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:21 smithi163 ceph-mon[87697]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:21.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:21 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:21.512 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:21 smithi163 ceph-mon[87697]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:10:21.512 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:21 smithi163 ceph-mon[87697]: osdmap e123: 8 total, 7 up, 8 in 2024-06-18T03:10:21.512 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:21 smithi163 podman[105568]: 2024-06-18 03:10:21.135237753 +0000 UTC m=+0.613798031 container remove 38b8f85b7dc05032f077d41f89aafed9831503352b332404a9605d317faa1583 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, name=centos-stream, release=754, ceph=True, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, version=8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64) 2024-06-18T03:10:21.512 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:21 smithi163 ceph-mon[82879]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:21.512 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:21 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:21.512 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:21 smithi163 ceph-mon[82879]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:10:21.512 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:21 smithi163 ceph-mon[82879]: osdmap e123: 8 total, 7 up, 8 in 2024-06-18T03:10:21.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:21 smithi186 ceph-mon[67751]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:10:21.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:21 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:21.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:21 smithi186 ceph-mon[67751]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:10:21.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:21 smithi186 ceph-mon[67751]: osdmap e123: 8 total, 7 up, 8 in 2024-06-18T03:10:21.793 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:21 smithi163 podman[105658]: 2024-06-18 03:10:21.414758672 +0000 UTC m=+0.017369317 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:10:21.793 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:21 smithi163 podman[105658]: 2024-06-18 03:10:21.545218046 +0000 UTC m=+0.147828683 container create 093108b9b3dc1aae483b606503f3157e6984209cd77aec4a7e7aab55cd694b46 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-deactivate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-06-18T03:10:21.793 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:21 smithi163 podman[105658]: 2024-06-18 03:10:21.755852359 +0000 UTC m=+0.358463009 container init 093108b9b3dc1aae483b606503f3157e6984209cd77aec4a7e7aab55cd694b46 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-deactivate, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240612) 2024-06-18T03:10:21.793 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:21 smithi163 podman[105658]: 2024-06-18 03:10:21.759215516 +0000 UTC m=+0.361826156 container start 093108b9b3dc1aae483b606503f3157e6984209cd77aec4a7e7aab55cd694b46 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-3781e3a, org.label-schema.build-date=20240612, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-06-18T03:10:22.195 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:21 smithi163 podman[105658]: 2024-06-18 03:10:21.839378547 +0000 UTC m=+0.441989197 container attach 093108b9b3dc1aae483b606503f3157e6984209cd77aec4a7e7aab55cd694b46 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:10:22.195 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:22 smithi163 podman[105658]: 2024-06-18 03:10:22.022442678 +0000 UTC m=+0.625053319 container died 093108b9b3dc1aae483b606503f3157e6984209cd77aec4a7e7aab55cd694b46 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240612, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-3781e3a, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-06-18T03:10:22.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:22 smithi163 ceph-mon[87697]: osdmap e124: 8 total, 7 up, 8 in 2024-06-18T03:10:22.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:22 smithi163 ceph-mon[82879]: osdmap e124: 8 total, 7 up, 8 in 2024-06-18T03:10:22.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:22 smithi186 ceph-mon[67751]: osdmap e124: 8 total, 7 up, 8 in 2024-06-18T03:10:23.042 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:22 smithi163 podman[105681]: 2024-06-18 03:10:22.760711001 +0000 UTC m=+0.732304216 container remove 093108b9b3dc1aae483b606503f3157e6984209cd77aec4a7e7aab55cd694b46 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-06-18T03:10:23.043 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:22 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.2.service: Deactivated successfully. 2024-06-18T03:10:23.043 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:22 smithi163 systemd[1]: Stopped Ceph osd.2 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:10:23.043 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:22 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.2.service: Consumed 12.603s CPU time. 2024-06-18T03:10:23.043 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:22 smithi163 systemd[1]: Starting Ceph osd.2 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:10:23.448 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:23 smithi163 ceph-mon[87697]: pgmap v116: 161 pgs: 6 peering, 12 stale+active+clean, 143 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-18T03:10:23.448 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:23 smithi163 ceph-mon[87697]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-18T03:10:23.448 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 podman[105783]: 2024-06-18 03:10:23.106822358 +0000 UTC m=+0.017697027 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:10:23.448 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 podman[105783]: 2024-06-18 03:10:23.212068958 +0000 UTC m=+0.122943627 container create c71affa409d39f82fcbb2adf5a764412ca24a4f6d2199e96c5b6496300f70f92 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-activate, ceph=True, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-3781e3a, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-06-18T03:10:23.448 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:23 smithi163 ceph-mon[82879]: pgmap v116: 161 pgs: 6 peering, 12 stale+active+clean, 143 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-18T03:10:23.449 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:23 smithi163 ceph-mon[82879]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-18T03:10:23.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:23 smithi186 ceph-mon[67751]: pgmap v116: 161 pgs: 6 peering, 12 stale+active+clean, 143 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-18T03:10:23.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:23 smithi186 ceph-mon[67751]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-18T03:10:23.792 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 podman[105783]: 2024-06-18 03:10:23.448226269 +0000 UTC m=+0.359100937 container init c71affa409d39f82fcbb2adf5a764412ca24a4f6d2199e96c5b6496300f70f92 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.vendor=CentOS) 2024-06-18T03:10:23.793 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 podman[105783]: 2024-06-18 03:10:23.452041186 +0000 UTC m=+0.362915855 container start c71affa409d39f82fcbb2adf5a764412ca24a4f6d2199e96c5b6496300f70f92 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-activate, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-06-18T03:10:23.793 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 podman[105783]: 2024-06-18 03:10:23.548403801 +0000 UTC m=+0.459278478 container attach c71affa409d39f82fcbb2adf5a764412ca24a4f6d2199e96c5b6496300f70f92 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-activate, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS) 2024-06-18T03:10:24.153 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:10:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:10:23] "GET /metrics HTTP/1.1" 200 38176 "" "Prometheus/2.43.0" 2024-06-18T03:10:24.153 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-activate[105798]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-18T03:10:24.153 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 bash[105783]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-18T03:10:24.153 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-activate[105798]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-18T03:10:24.153 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 bash[105783]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-18T03:10:24.153 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-activate[105798]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-18T03:10:24.153 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 bash[105783]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-18T03:10:24.153 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-activate[105798]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-18T03:10:24.154 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 bash[105783]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-18T03:10:24.154 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-activate[105798]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-06-18T03:10:24.154 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 bash[105783]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-06-18T03:10:24.154 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-activate[105798]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-18T03:10:24.154 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 bash[105783]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-18T03:10:24.154 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-activate[105798]: --> ceph-volume raw activate successful for osd ID: 2 2024-06-18T03:10:24.154 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 bash[105783]: --> ceph-volume raw activate successful for osd ID: 2 2024-06-18T03:10:24.154 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:23 smithi163 podman[105783]: 2024-06-18 03:10:23.871862979 +0000 UTC m=+0.782737651 container died c71affa409d39f82fcbb2adf5a764412ca24a4f6d2199e96c5b6496300f70f92 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-activate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:10:24.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:24 smithi163 ceph-mon[87697]: Health check failed: Degraded data redundancy: 6/723 objects degraded (0.830%), 1 pg degraded (PG_DEGRADED) 2024-06-18T03:10:24.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:24 smithi163 ceph-mon[82879]: Health check failed: Degraded data redundancy: 6/723 objects degraded (0.830%), 1 pg degraded (PG_DEGRADED) 2024-06-18T03:10:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:24 smithi186 ceph-mon[67751]: Health check failed: Degraded data redundancy: 6/723 objects degraded (0.830%), 1 pg degraded (PG_DEGRADED) 2024-06-18T03:10:24.812 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:24 smithi163 podman[105912]: 2024-06-18 03:10:24.557523074 +0000 UTC m=+0.679907139 container remove c71affa409d39f82fcbb2adf5a764412ca24a4f6d2199e96c5b6496300f70f92 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3781e3a, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0) 2024-06-18T03:10:24.812 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:24 smithi163 podman[105973]: 2024-06-18 03:10:24.676881175 +0000 UTC m=+0.017064398 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:10:25.123 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:24 smithi163 podman[105973]: 2024-06-18 03:10:24.813025251 +0000 UTC m=+0.153208505 container create 05782842a9a41071ab86edecaaca7da717f8f4c226a1f2befbfb9a1eb37cfdfc (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:10:25.123 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:25 smithi163 podman[105973]: 2024-06-18 03:10:25.031871707 +0000 UTC m=+0.372054923 container init 05782842a9a41071ab86edecaaca7da717f8f4c226a1f2befbfb9a1eb37cfdfc (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-18T03:10:25.123 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:25 smithi163 podman[105973]: 2024-06-18 03:10:25.035161195 +0000 UTC m=+0.375344404 container start 05782842a9a41071ab86edecaaca7da717f8f4c226a1f2befbfb9a1eb37cfdfc (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-3781e3a, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240612, ceph=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-06-18T03:10:25.123 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:25 smithi163 ceph-osd[105991]: -- 172.21.15.163:0/1233643694 <== mon.0 v2:172.21.15.163:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5637d5ad3a40 con 0x5637d6890400 2024-06-18T03:10:25.542 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:25 smithi163 bash[105973]: 05782842a9a41071ab86edecaaca7da717f8f4c226a1f2befbfb9a1eb37cfdfc 2024-06-18T03:10:25.543 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:25 smithi163 systemd[1]: Started Ceph osd.2 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:10:25.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:25 smithi163 ceph-mon[82879]: pgmap v117: 161 pgs: 6 active+undersized, 6 peering, 10 stale+active+clean, 1 active+undersized+degraded, 138 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 6/723 objects degraded (0.830%) 2024-06-18T03:10:25.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:25.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:25.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:25.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:25 smithi163 ceph-mon[87697]: pgmap v117: 161 pgs: 6 active+undersized, 6 peering, 10 stale+active+clean, 1 active+undersized+degraded, 138 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 6/723 objects degraded (0.830%) 2024-06-18T03:10:25.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:25.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:25.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:25.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:25 smithi186 ceph-mon[67751]: pgmap v117: 161 pgs: 6 active+undersized, 6 peering, 10 stale+active+clean, 1 active+undersized+degraded, 138 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 6/723 objects degraded (0.830%) 2024-06-18T03:10:25.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:25.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:25.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:25.915 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2[105987]: 2024-06-18T03:10:25.574+0000 7f4ba6133640 -1 Falling back to public interface 2024-06-18T03:10:26.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:26 smithi186 ceph-mon[67751]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-18T03:10:26.714 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:26 smithi163 ceph-mon[87697]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-18T03:10:26.714 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:26 smithi163 ceph-mon[82879]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-18T03:10:27.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:27 smithi186 ceph-mon[67751]: pgmap v118: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-06-18T03:10:27.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:27 smithi163 ceph-mon[87697]: pgmap v118: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-06-18T03:10:27.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:27 smithi163 ceph-mon[82879]: pgmap v118: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-06-18T03:10:28.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:28 smithi163 ceph-mon[82879]: from='osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-18T03:10:28.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:28 smithi163 ceph-mon[87697]: from='osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-18T03:10:28.543 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2[105987]: 2024-06-18T03:10:28.268+0000 7f4ba6133640 -1 osd.2 0 read_superblock omap replica is missing. 2024-06-18T03:10:28.543 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2[105987]: 2024-06-18T03:10:28.288+0000 7f4ba6133640 -1 osd.2 122 log_to_monitors true 2024-06-18T03:10:28.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:28 smithi186 ceph-mon[67751]: from='osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-18T03:10:28.856 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:10:28 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2[105987]: 2024-06-18T03:10:28.787+0000 7f4b99ed6640 -1 osd.2 122 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T03:10:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:29 smithi186 ceph-mon[67751]: pgmap v119: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-06-18T03:10:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:29 smithi186 ceph-mon[67751]: from='osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-18T03:10:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:29 smithi186 ceph-mon[67751]: osdmap e125: 8 total, 7 up, 8 in 2024-06-18T03:10:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:29 smithi186 ceph-mon[67751]: from='osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:10:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:10:29.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:29 smithi186 ceph-mon[67751]: from='osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472]' entity='osd.2' 2024-06-18T03:10:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[87697]: pgmap v119: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-06-18T03:10:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[87697]: from='osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-18T03:10:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[87697]: osdmap e125: 8 total, 7 up, 8 in 2024-06-18T03:10:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[87697]: from='osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:10:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:10:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[87697]: from='osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472]' entity='osd.2' 2024-06-18T03:10:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[82879]: pgmap v119: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-06-18T03:10:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[82879]: from='osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-18T03:10:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[82879]: osdmap e125: 8 total, 7 up, 8 in 2024-06-18T03:10:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[82879]: from='osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:10:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:10:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:29 smithi163 ceph-mon[82879]: from='osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472]' entity='osd.2' 2024-06-18T03:10:30.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:30.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:30.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[82879]: pgmap v121: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 369 B/s rd, 0 op/s; 66/723 objects degraded (9.129%) 2024-06-18T03:10:30.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[82879]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:10:30.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[82879]: osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472] boot 2024-06-18T03:10:30.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[82879]: osdmap e126: 8 total, 8 up, 8 in 2024-06-18T03:10:30.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T03:10:30.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:30.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:30.681 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:30.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:30.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:30.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[87697]: pgmap v121: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 369 B/s rd, 0 op/s; 66/723 objects degraded (9.129%) 2024-06-18T03:10:30.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[87697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:10:30.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[87697]: osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472] boot 2024-06-18T03:10:30.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[87697]: osdmap e126: 8 total, 8 up, 8 in 2024-06-18T03:10:30.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T03:10:30.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:30.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:30.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:30 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:30 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:30.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:30 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:30.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:30 smithi186 ceph-mon[67751]: pgmap v121: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 369 B/s rd, 0 op/s; 66/723 objects degraded (9.129%) 2024-06-18T03:10:30.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:30 smithi186 ceph-mon[67751]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:10:30.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:30 smithi186 ceph-mon[67751]: osd.2 [v2:172.21.15.163:6818/2411915472,v1:172.21.15.163:6819/2411915472] boot 2024-06-18T03:10:30.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:30 smithi186 ceph-mon[67751]: osdmap e126: 8 total, 8 up, 8 in 2024-06-18T03:10:30.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:30 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-18T03:10:30.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:30 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:30.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:30 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:30.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:30 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:32.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:31 smithi163 ceph-mon[82879]: osdmap e127: 8 total, 8 up, 8 in 2024-06-18T03:10:32.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:31 smithi163 ceph-mon[87697]: osdmap e127: 8 total, 8 up, 8 in 2024-06-18T03:10:32.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:31 smithi186 ceph-mon[67751]: osdmap e127: 8 total, 8 up, 8 in 2024-06-18T03:10:33.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:32 smithi163 ceph-mon[87697]: pgmap v124: 161 pgs: 5 peering, 30 active+undersized, 12 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 44/723 objects degraded (6.086%) 2024-06-18T03:10:33.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:32 smithi163 ceph-mon[87697]: Health check update: Degraded data redundancy: 44/723 objects degraded (6.086%), 12 pgs degraded (PG_DEGRADED) 2024-06-18T03:10:33.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:32 smithi163 ceph-mon[82879]: pgmap v124: 161 pgs: 5 peering, 30 active+undersized, 12 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 44/723 objects degraded (6.086%) 2024-06-18T03:10:33.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:32 smithi163 ceph-mon[82879]: Health check update: Degraded data redundancy: 44/723 objects degraded (6.086%), 12 pgs degraded (PG_DEGRADED) 2024-06-18T03:10:33.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:32 smithi186 ceph-mon[67751]: pgmap v124: 161 pgs: 5 peering, 30 active+undersized, 12 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 44/723 objects degraded (6.086%) 2024-06-18T03:10:33.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:32 smithi186 ceph-mon[67751]: Health check update: Degraded data redundancy: 44/723 objects degraded (6.086%), 12 pgs degraded (PG_DEGRADED) 2024-06-18T03:10:34.197 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:10:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:10:33] "GET /metrics HTTP/1.1" 200 38030 "" "Prometheus/2.43.0" 2024-06-18T03:10:34.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.197 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:34.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:10:34.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: pgmap v125: 161 pgs: 5 peering, 26 active+undersized, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 898 B/s rd, 0 op/s; 38/723 objects degraded (5.256%) 2024-06-18T03:10:34.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: pgmap v126: 161 pgs: 5 peering, 26 active+undersized, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 38/723 objects degraded (5.256%) 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: Upgrade: Finalizing container_image settings 2024-06-18T03:10:34.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:10:34.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:10:34.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:10:34.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:10:34.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:10:34.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:10:34.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:34.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:10:34.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: pgmap v125: 161 pgs: 5 peering, 26 active+undersized, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 898 B/s rd, 0 op/s; 38/723 objects degraded (5.256%) 2024-06-18T03:10:34.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: pgmap v126: 161 pgs: 5 peering, 26 active+undersized, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 38/723 objects degraded (5.256%) 2024-06-18T03:10:34.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:10:34.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:10:34.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: Upgrade: Finalizing container_image settings 2024-06-18T03:10:34.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:10:34.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:10:34.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:10:34.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:10:34.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:10:34.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:10:34.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:10:34.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:10:34.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:10:34.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:10:34.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:10:34.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:10:34.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:10:34.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:10:34.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:10:34.548 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:10:34.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:34.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:10:34.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:34.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:10:34.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:10:34.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:10:34.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:10:34.550 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:10:34.551 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:34.551 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:10:34.551 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.551 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.551 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:34.551 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:10:34.551 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: pgmap v125: 161 pgs: 5 peering, 26 active+undersized, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 898 B/s rd, 0 op/s; 38/723 objects degraded (5.256%) 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: pgmap v126: 161 pgs: 5 peering, 26 active+undersized, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 38/723 objects degraded (5.256%) 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:34.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: Upgrade: Finalizing container_image settings 2024-06-18T03:10:34.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:10:34.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:10:34.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:10:34.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:10:34.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:10:34.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:10:34.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:10:34.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:10:34.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:10:34.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:10:34.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:10:34.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:10:34.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:10:34.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:10:34.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:10:34.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:10:34.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:10:34.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:34.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:10:34.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:34.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:10:34.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:10:34.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:35.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:35 smithi163 ceph-mon[82879]: Upgrade: Complete! 2024-06-18T03:10:35.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:35 smithi163 ceph-mon[82879]: pgmap v127: 161 pgs: 5 peering, 15 active+undersized, 9 active+undersized+degraded, 132 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-06-18T03:10:35.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:35 smithi163 ceph-mon[82879]: pgmap v128: 161 pgs: 5 peering, 15 active+undersized, 9 active+undersized+degraded, 132 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-06-18T03:10:35.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:35 smithi163 ceph-mon[82879]: pgmap v129: 161 pgs: 5 peering, 15 active+undersized, 9 active+undersized+degraded, 132 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 2 op/s; 35/723 objects degraded (4.841%) 2024-06-18T03:10:35.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:35 smithi163 ceph-mon[82879]: pgmap v130: 161 pgs: 5 peering, 15 active+undersized, 9 active+undersized+degraded, 132 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 3 op/s; 35/723 objects degraded (4.841%) 2024-06-18T03:10:35.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:35 smithi163 ceph-mon[87697]: Upgrade: Complete! 2024-06-18T03:10:35.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:35 smithi163 ceph-mon[87697]: pgmap v127: 161 pgs: 5 peering, 15 active+undersized, 9 active+undersized+degraded, 132 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-06-18T03:10:35.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:35 smithi163 ceph-mon[87697]: pgmap v128: 161 pgs: 5 peering, 15 active+undersized, 9 active+undersized+degraded, 132 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-06-18T03:10:35.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:35 smithi163 ceph-mon[87697]: pgmap v129: 161 pgs: 5 peering, 15 active+undersized, 9 active+undersized+degraded, 132 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 2 op/s; 35/723 objects degraded (4.841%) 2024-06-18T03:10:35.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:35 smithi163 ceph-mon[87697]: pgmap v130: 161 pgs: 5 peering, 15 active+undersized, 9 active+undersized+degraded, 132 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 3 op/s; 35/723 objects degraded (4.841%) 2024-06-18T03:10:35.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:35 smithi186 ceph-mon[67751]: Upgrade: Complete! 2024-06-18T03:10:35.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:35 smithi186 ceph-mon[67751]: pgmap v127: 161 pgs: 5 peering, 15 active+undersized, 9 active+undersized+degraded, 132 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-06-18T03:10:35.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:35 smithi186 ceph-mon[67751]: pgmap v128: 161 pgs: 5 peering, 15 active+undersized, 9 active+undersized+degraded, 132 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 35/723 objects degraded (4.841%) 2024-06-18T03:10:35.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:35 smithi186 ceph-mon[67751]: pgmap v129: 161 pgs: 5 peering, 15 active+undersized, 9 active+undersized+degraded, 132 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 2 op/s; 35/723 objects degraded (4.841%) 2024-06-18T03:10:35.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:35 smithi186 ceph-mon[67751]: pgmap v130: 161 pgs: 5 peering, 15 active+undersized, 9 active+undersized+degraded, 132 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 3 op/s; 35/723 objects degraded (4.841%) 2024-06-18T03:10:36.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:36 smithi163 ceph-mon[87697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/723 objects degraded (4.841%), 9 pgs degraded) 2024-06-18T03:10:36.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:36 smithi163 ceph-mon[82879]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/723 objects degraded (4.841%), 9 pgs degraded) 2024-06-18T03:10:36.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:36 smithi186 ceph-mon[67751]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/723 objects degraded (4.841%), 9 pgs degraded) 2024-06-18T03:10:37.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:37 smithi163 ceph-mon[87697]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 5 op/s 2024-06-18T03:10:37.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:37 smithi163 ceph-mon[82879]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 5 op/s 2024-06-18T03:10:37.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:37 smithi186 ceph-mon[67751]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 5 op/s 2024-06-18T03:10:39.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:39 smithi163 ceph-mon[87697]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s 2024-06-18T03:10:39.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:39 smithi163 ceph-mon[82879]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s 2024-06-18T03:10:39.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:39 smithi186 ceph-mon[67751]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s 2024-06-18T03:10:41.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:41 smithi163 ceph-mon[87697]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-06-18T03:10:41.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:41 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:41.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:41 smithi163 ceph-mon[82879]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-06-18T03:10:41.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:41 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:41.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:41 smithi186 ceph-mon[67751]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-06-18T03:10:41.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:41 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:43.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:43 smithi163 ceph-mon[87697]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:43.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:43 smithi163 ceph-mon[82879]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:43.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:43 smithi186 ceph-mon[67751]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:44.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:10:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:10:43] "GET /metrics HTTP/1.1" 200 38187 "" "Prometheus/2.43.0" 2024-06-18T03:10:44.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:44 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:44.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:44 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:10:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:44 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:44 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:10:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:44 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:10:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:44 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:10:45.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:45 smithi163 ceph-mon[82879]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:10:45.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:45 smithi163 ceph-mon[87697]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:10:45.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:45 smithi186 ceph-mon[67751]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:10:46.155 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps' 2024-06-18T03:10:47.430 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:47 smithi163 ceph-mon[87697]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:47.430 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:47 smithi163 ceph-mon[87697]: from='client.44364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:47.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:47 smithi163 ceph-mon[82879]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:47.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:47 smithi163 ceph-mon[82879]: from='client.44364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:47 smithi186 ceph-mon[67751]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:47 smithi186 ceph-mon[67751]: from='client.44364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:48.134 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:10:48.134 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (5m) 18s ago 27m 27.7M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (5m) 3m ago 27m 76.6M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 18s ago 26m 59.0M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (6m) 3m ago 31m 438M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (16m) 18s ago 33m 500M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (3m) 18s ago 33m 49.9M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (4m) 3m ago 32m 41.4M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (3m) 18s ago 32m 42.6M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (6m) 18s ago 28m 16.1M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (6m) 3m ago 28m 15.7M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (110s) 18s ago 31m 44.2M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (76s) 18s ago 31m 43.4M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (23s) 18s ago 30m 13.2M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 05782842a9a4 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (30m) 18s ago 30m 56.5M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:10:48.135 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (30m) 3m ago 30m 61.6M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:10:48.136 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (29m) 3m ago 29m 52.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:10:48.136 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (29m) 3m ago 29m 52.2M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:10:48.136 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (28m) 3m ago 28m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:10:48.136 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (5m) 3m ago 28m 69.4M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:10:48.136 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (26m) 18s ago 26m 80.7M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:10:48.136 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (26m) 3m ago 26m 80.6M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:10:48.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-06-18T03:10:49.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:49 smithi163 ceph-mon[87697]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:49.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:49 smithi163 ceph-mon[87697]: from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:49.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:49 smithi163 ceph-mon[82879]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:49.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:49 smithi163 ceph-mon[82879]: from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:49.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:49 smithi186 ceph-mon[67751]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:49.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:49 smithi186 ceph-mon[67751]: from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:50.651 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:10:51.277 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-06-18T03:10:51.654 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:51 smithi163 ceph-mon[82879]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:51.654 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:51 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:51.655 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:51 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/2842342216' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:51.655 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:51 smithi163 ceph-mon[87697]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:51.655 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:51 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:51.655 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:51 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/2842342216' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:51.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:51 smithi186 ceph-mon[67751]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:51.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:51 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:10:51.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:51 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/2842342216' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:10:53.603 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:53 smithi163 ceph-mon[87697]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:53.603 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:53 smithi163 ceph-mon[82879]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:53.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:53 smithi186 ceph-mon[67751]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:54.146 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:10:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:10:53] "GET /metrics HTTP/1.1" 200 38187 "" "Prometheus/2.43.0" 2024-06-18T03:10:54.404 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:10:54.404 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:54 smithi163 ceph-mon[87697]: from='client.44382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:54.405 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:54 smithi163 ceph-mon[82879]: from='client.44382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:54 smithi186 ceph-mon[67751]: from='client.44382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:55.042 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade status' 2024-06-18T03:10:55.598 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:55 smithi163 ceph-mon[87697]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:55.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:55 smithi163 ceph-mon[82879]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:55.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:55 smithi186 ceph-mon[67751]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:10:56.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:56 smithi163 ceph-mon[82879]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:56.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:56 smithi163 ceph-mon[87697]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:56 smithi186 ceph-mon[67751]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:10:57.019 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:10:57.019 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": null, 2024-06-18T03:10:57.019 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": false, 2024-06-18T03:10:57.019 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "", 2024-06-18T03:10:57.019 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:10:57.019 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": null, 2024-06-18T03:10:57.019 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "", 2024-06-18T03:10:57.019 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:10:57.020 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:10:57.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:57 smithi163 ceph-mon[87697]: from='client.44388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:57.633 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:57 smithi163 ceph-mon[82879]: from='client.44388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:57.657 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph health detail' 2024-06-18T03:10:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:57 smithi186 ceph-mon[67751]: from='client.44388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:10:58.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:58 smithi163 ceph-mon[82879]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:10:58.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:10:58 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:10:58.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:58 smithi163 ceph-mon[87697]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:10:58.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:10:58 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:10:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:58 smithi186 ceph-mon[67751]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:10:58.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:10:58 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:10:59.680 INFO:teuthology.orchestra.run.smithi163.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:10:59.680 INFO:teuthology.orchestra.run.smithi163.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:10:59.680 INFO:teuthology.orchestra.run.smithi163.stdout: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in error state 2024-06-18T03:11:00.248 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-06-18T03:11:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:00 smithi186 ceph-mon[67751]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:00 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/1364378276' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:11:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:00 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:00.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:00 smithi163 ceph-mon[87697]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:00.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:00 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/1364378276' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:11:00.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:00 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:00.852 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:00 smithi163 ceph-mon[82879]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:00.852 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:00 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/1364378276' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:11:00.852 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:00 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:02 smithi186 ceph-mon[67751]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:11:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:02 smithi186 ceph-mon[67751]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:02.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:02 smithi163 ceph-mon[87697]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:11:02.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:02 smithi163 ceph-mon[87697]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:02.862 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:02 smithi163 ceph-mon[82879]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:11:02.862 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:02 smithi163 ceph-mon[82879]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:03.726 INFO:teuthology.orchestra.run.smithi163.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:11:04.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:11:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:11:03] "GET /metrics HTTP/1.1" 200 38183 "" "Prometheus/2.43.0" 2024-06-18T03:11:04.518 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-18T03:11:04.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[82879]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:11:04.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[82879]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:11:04.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:04.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:04.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:11:04.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[82879]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-06-18T03:11:04.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[82879]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-06-18T03:11:04.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:04.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[82879]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:11:04.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[87697]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:11:04.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[87697]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:11:04.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:04.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:04.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:11:04.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[87697]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-06-18T03:11:04.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[87697]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-06-18T03:11:04.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:04.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:04 smithi163 ceph-mon[87697]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:11:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:04 smithi186 ceph-mon[67751]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:11:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:04 smithi186 ceph-mon[67751]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:11:05.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:11:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:04 smithi186 ceph-mon[67751]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-06-18T03:11:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:04 smithi186 ceph-mon[67751]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-06-18T03:11:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:05.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:04 smithi186 ceph-mon[67751]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:11:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:06.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:11:06.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:11:06.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:11:06.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:06.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mgr 2024-06-18T03:11:06.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:06.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:06.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mon 2024-06-18T03:11:06.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:06.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:06.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all crash 2024-06-18T03:11:06.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:06.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-18T03:11:06.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-18T03:11:06.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: Upgrade: osd.3 is safe to restart 2024-06-18T03:11:06.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:06 smithi186 ceph-mon[67751]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:06.919 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:06.919 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mgr 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mon 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all crash 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-18T03:11:06.920 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: Upgrade: osd.3 is safe to restart 2024-06-18T03:11:06.921 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[87697]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:06.921 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:06.921 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:11:06.921 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:11:06.921 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:11:06.921 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:06.921 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mgr 2024-06-18T03:11:06.921 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:06.921 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:06.921 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mon 2024-06-18T03:11:06.921 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:06.922 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:06.922 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all crash 2024-06-18T03:11:06.922 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:06.922 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-18T03:11:06.922 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-18T03:11:06.922 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: Upgrade: osd.3 is safe to restart 2024-06-18T03:11:06.922 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:06 smithi163 ceph-mon[82879]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:07.534 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:11:08.206 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:11:08.207 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (5m) 38s ago 27m 27.7M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:11:08.207 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (5m) 3m ago 27m 76.6M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:11:08.207 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 38s ago 26m 59.0M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:11:08.207 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (6m) 3m ago 32m 438M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:11:08.207 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (16m) 38s ago 34m 500M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:11:08.207 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (3m) 38s ago 34m 49.9M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:11:08.207 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (4m) 3m ago 32m 41.4M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:11:08.207 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (3m) 38s ago 32m 42.6M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:11:08.207 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (6m) 38s ago 28m 16.1M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:11:08.207 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (6m) 3m ago 28m 15.7M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:11:08.207 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (2m) 38s ago 32m 44.2M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:11:08.207 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (96s) 38s ago 31m 43.4M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:11:08.207 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (43s) 38s ago 31m 13.2M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 05782842a9a4 2024-06-18T03:11:08.208 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (30m) 38s ago 30m 56.5M 2992M 17.2.0 e1d6a67b021e 88baff0ebd1c 2024-06-18T03:11:08.208 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (30m) 3m ago 30m 61.6M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:11:08.208 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (30m) 3m ago 30m 52.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:11:08.208 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (29m) 3m ago 29m 52.2M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:11:08.208 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (29m) 3m ago 29m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:11:08.208 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (6m) 3m ago 28m 69.4M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:11:08.208 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (27m) 38s ago 27m 80.7M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:11:08.208 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (27m) 3m ago 27m 80.6M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:11:08.618 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:11:08.618 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:11:08.618 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:11:08.618 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:11:08.618 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:11:08.618 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:11:08.619 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:11:08.619 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:11:08.619 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-06-18T03:11:08.619 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:11:08.619 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:11:08.620 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:11:08.620 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:11:08.620 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:11:08.620 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:11:08.620 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-06-18T03:11:08.620 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 8 2024-06-18T03:11:08.621 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:11:08.621 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:11:08.970 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:11:08.970 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:11:08.970 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:11:08.970 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-06-18T03:11:08.970 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:11:08.970 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "3/8 daemons upgraded", 2024-06-18T03:11:08.971 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading osd daemons", 2024-06-18T03:11:08.971 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:11:08.971 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:11:09.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[87697]: from='client.44406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:09.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[87697]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-06-18T03:11:09.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[87697]: from='client.44412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:09.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[87697]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:09.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:09.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-18T03:11:09.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:09.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/2447702741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:09.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[82879]: from='client.44406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:09.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[82879]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-06-18T03:11:09.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[82879]: from='client.44412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:09.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[82879]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:09.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:09.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-18T03:11:09.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:09.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:08 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/2447702741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:08 smithi186 ceph-mon[67751]: from='client.44406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:08 smithi186 ceph-mon[67751]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-06-18T03:11:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:08 smithi186 ceph-mon[67751]: from='client.44412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:08 smithi186 ceph-mon[67751]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:09.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:08 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:09.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:08 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-18T03:11:09.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:08 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:09.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:08 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/2447702741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:10.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:09 smithi163 ceph-mon[87697]: Upgrade: Updating osd.3 2024-06-18T03:11:10.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:09 smithi163 ceph-mon[87697]: Deploying daemon osd.3 on smithi163 2024-06-18T03:11:10.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:09 smithi163 ceph-mon[87697]: from='client.54397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:10.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:09 smithi163 ceph-mon[82879]: Upgrade: Updating osd.3 2024-06-18T03:11:10.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:09 smithi163 ceph-mon[82879]: Deploying daemon osd.3 on smithi163 2024-06-18T03:11:10.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:09 smithi163 ceph-mon[82879]: from='client.54397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:10.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:09 smithi186 ceph-mon[67751]: Upgrade: Updating osd.3 2024-06-18T03:11:10.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:09 smithi186 ceph-mon[67751]: Deploying daemon osd.3 on smithi163 2024-06-18T03:11:10.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:09 smithi186 ceph-mon[67751]: from='client.54397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:11.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:10 smithi163 ceph-mon[87697]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:11.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:10 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:11.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:10 smithi163 ceph-mon[82879]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:11.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:10 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:11.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:10 smithi186 ceph-mon[67751]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:11.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:10 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:11.543 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:11 smithi163 systemd[1]: Stopping Ceph osd.3 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:11:12.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:11 smithi163 ceph-mon[87697]: osd.3 marked itself down and dead 2024-06-18T03:11:12.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:11 smithi163 ceph-mon[82879]: osd.3 marked itself down and dead 2024-06-18T03:11:12.043 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3[41430]: 2024-06-18T03:11:11.579+0000 7f94e10be700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:11:12.043 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3[41430]: 2024-06-18T03:11:11.579+0000 7f94e10be700 -1 osd.3 127 *** Got signal Terminated *** 2024-06-18T03:11:12.043 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3[41430]: 2024-06-18T03:11:11.579+0000 7f94e10be700 -1 osd.3 127 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:11:12.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:11 smithi186 ceph-mon[67751]: osd.3 marked itself down and dead 2024-06-18T03:11:12.543 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:12 smithi163 podman[111975]: 2024-06-18 03:11:12.366806528 +0000 UTC m=+0.875153605 container died 88baff0ebd1c83f4bb4924409f06f5fa0870901b336d58570630ce095eaca5e6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, architecture=x86_64, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-06-18T03:11:13.023 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:12 smithi163 ceph-mon[87697]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:13.023 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:12 smithi163 ceph-mon[87697]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:11:13.023 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:12 smithi163 ceph-mon[87697]: osdmap e128: 8 total, 7 up, 8 in 2024-06-18T03:11:13.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:12 smithi163 ceph-mon[82879]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:13.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:12 smithi163 ceph-mon[82879]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:11:13.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:12 smithi163 ceph-mon[82879]: osdmap e128: 8 total, 7 up, 8 in 2024-06-18T03:11:13.023 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:12 smithi163 podman[111975]: 2024-06-18 03:11:12.763105915 +0000 UTC m=+1.271453038 container cleanup 88baff0ebd1c83f4bb4924409f06f5fa0870901b336d58570630ce095eaca5e6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, vcs-type=git, io.openshift.expose-services=, distribution-scope=public, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container) 2024-06-18T03:11:13.024 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:12 smithi163 bash[111975]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3 2024-06-18T03:11:13.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:12 smithi186 ceph-mon[67751]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:13.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:12 smithi186 ceph-mon[67751]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:11:13.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:12 smithi186 ceph-mon[67751]: osdmap e128: 8 total, 7 up, 8 in 2024-06-18T03:11:13.292 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:13 smithi163 podman[111993]: 2024-06-18 03:11:13.023032731 +0000 UTC m=+0.659827061 container remove 88baff0ebd1c83f4bb4924409f06f5fa0870901b336d58570630ce095eaca5e6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, version=8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, architecture=x86_64, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., ceph=True, release=754, com.redhat.component=centos-stream-container) 2024-06-18T03:11:13.604 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:13 smithi163 podman[112080]: 2024-06-18 03:11:13.242506467 +0000 UTC m=+0.016971974 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:11:13.604 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:13 smithi163 podman[112080]: 2024-06-18 03:11:13.365979942 +0000 UTC m=+0.140445492 container create 168ecb44e2e14e6e97544d6840e5aa49f6bca3213522a412e965446e22797154 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-06-18T03:11:13.868 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:13 smithi163 ceph-mon[82879]: osdmap e129: 8 total, 7 up, 8 in 2024-06-18T03:11:13.869 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:13.869 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:11:13.869 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:13 smithi163 ceph-mon[87697]: osdmap e129: 8 total, 7 up, 8 in 2024-06-18T03:11:13.869 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:13.869 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:11:13.869 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:11:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:11:13] "GET /metrics HTTP/1.1" 200 38186 "" "Prometheus/2.43.0" 2024-06-18T03:11:13.870 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:13 smithi163 podman[112080]: 2024-06-18 03:11:13.604254779 +0000 UTC m=+0.378720286 container init 168ecb44e2e14e6e97544d6840e5aa49f6bca3213522a412e965446e22797154 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-06-18T03:11:13.870 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:13 smithi163 podman[112080]: 2024-06-18 03:11:13.608541976 +0000 UTC m=+0.383007494 container start 168ecb44e2e14e6e97544d6840e5aa49f6bca3213522a412e965446e22797154 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:11:13.870 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:13 smithi163 podman[112080]: 2024-06-18 03:11:13.679145463 +0000 UTC m=+0.453610990 container attach 168ecb44e2e14e6e97544d6840e5aa49f6bca3213522a412e965446e22797154 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-06-18T03:11:13.870 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:13 smithi163 podman[112080]: 2024-06-18 03:11:13.868843215 +0000 UTC m=+0.643308725 container died 168ecb44e2e14e6e97544d6840e5aa49f6bca3213522a412e965446e22797154 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-deactivate, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-3781e3a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-06-18T03:11:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:13 smithi186 ceph-mon[67751]: osdmap e129: 8 total, 7 up, 8 in 2024-06-18T03:11:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:11:14.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:14 smithi163 podman[112103]: 2024-06-18 03:11:14.491939503 +0000 UTC m=+0.617146013 container remove 168ecb44e2e14e6e97544d6840e5aa49f6bca3213522a412e965446e22797154 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-deactivate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-06-18T03:11:14.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:14 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.3.service: Deactivated successfully. 2024-06-18T03:11:14.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:14 smithi163 systemd[1]: Stopped Ceph osd.3 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:11:14.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:14 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.3.service: Consumed 16.377s CPU time. 2024-06-18T03:11:14.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:14 smithi163 systemd[1]: Starting Ceph osd.3 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:11:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:14 smithi186 ceph-mon[67751]: pgmap v154: 161 pgs: 6 active+undersized, 14 peering, 20 stale+active+clean, 121 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-06-18T03:11:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:14 smithi186 ceph-mon[67751]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-06-18T03:11:15.120 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:14 smithi163 ceph-mon[87697]: pgmap v154: 161 pgs: 6 active+undersized, 14 peering, 20 stale+active+clean, 121 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-06-18T03:11:15.121 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:14 smithi163 ceph-mon[87697]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-06-18T03:11:15.121 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:14 smithi163 ceph-mon[82879]: pgmap v154: 161 pgs: 6 active+undersized, 14 peering, 20 stale+active+clean, 121 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-06-18T03:11:15.121 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:14 smithi163 ceph-mon[82879]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-06-18T03:11:15.121 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:14 smithi163 podman[112203]: 2024-06-18 03:11:14.92633781 +0000 UTC m=+0.111068171 container create ec41624fa9d8395713ada4c763390a3154aa86164b78f2d2f779e2697f3ac1a4 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-06-18T03:11:15.121 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:14 smithi163 podman[112203]: 2024-06-18 03:11:14.832981311 +0000 UTC m=+0.017711631 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:11:15.121 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 podman[112203]: 2024-06-18 03:11:15.121091787 +0000 UTC m=+0.305822120 container init ec41624fa9d8395713ada4c763390a3154aa86164b78f2d2f779e2697f3ac1a4 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-activate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, io.buildah.version=1.36.0) 2024-06-18T03:11:15.490 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 podman[112203]: 2024-06-18 03:11:15.124742599 +0000 UTC m=+0.309472927 container start ec41624fa9d8395713ada4c763390a3154aa86164b78f2d2f779e2697f3ac1a4 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-activate, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, ceph=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:11:15.490 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 podman[112203]: 2024-06-18 03:11:15.204156269 +0000 UTC m=+0.388886597 container attach ec41624fa9d8395713ada4c763390a3154aa86164b78f2d2f779e2697f3ac1a4 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2) 2024-06-18T03:11:15.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-activate[112218]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-18T03:11:15.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 bash[112203]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-18T03:11:15.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-activate[112218]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-18T03:11:15.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 bash[112203]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-18T03:11:15.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-activate[112218]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-18T03:11:15.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 bash[112203]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-18T03:11:15.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-activate[112218]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-18T03:11:15.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 bash[112203]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-18T03:11:15.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-activate[112218]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-06-18T03:11:15.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 bash[112203]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-06-18T03:11:15.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-activate[112218]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-18T03:11:15.794 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 bash[112203]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-18T03:11:15.794 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-activate[112218]: --> ceph-volume raw activate successful for osd ID: 3 2024-06-18T03:11:15.794 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 bash[112203]: --> ceph-volume raw activate successful for osd ID: 3 2024-06-18T03:11:15.794 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:15 smithi163 podman[112203]: 2024-06-18 03:11:15.544839252 +0000 UTC m=+0.729569574 container died ec41624fa9d8395713ada4c763390a3154aa86164b78f2d2f779e2697f3ac1a4 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-18T03:11:16.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:15 smithi163 ceph-mon[87697]: Health check failed: Degraded data redundancy: 68/723 objects degraded (9.405%), 19 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:16.267 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:15 smithi163 ceph-mon[82879]: Health check failed: Degraded data redundancy: 68/723 objects degraded (9.405%), 19 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:16.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:15 smithi186 ceph-mon[67751]: Health check failed: Degraded data redundancy: 68/723 objects degraded (9.405%), 19 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:16.542 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:16 smithi163 podman[112342]: 2024-06-18 03:11:16.267152311 +0000 UTC m=+0.712801779 container remove ec41624fa9d8395713ada4c763390a3154aa86164b78f2d2f779e2697f3ac1a4 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=squid-3781e3a, ceph=True, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-18T03:11:16.543 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:16 smithi163 podman[112401]: 2024-06-18 03:11:16.383875099 +0000 UTC m=+0.017688620 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:11:16.543 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:16 smithi163 podman[112401]: 2024-06-18 03:11:16.501435557 +0000 UTC m=+0.135249102 container create 526499193ea3c0b3d7d0921b84721ef025a3efe76cd7e1a010f62d7b5c775548 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_CLEAN=True, ceph=True) 2024-06-18T03:11:16.867 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:16 smithi163 podman[112401]: 2024-06-18 03:11:16.696113723 +0000 UTC m=+0.329927238 container init 526499193ea3c0b3d7d0921b84721ef025a3efe76cd7e1a010f62d7b5c775548 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-06-18T03:11:16.868 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:16 smithi163 podman[112401]: 2024-06-18 03:11:16.699898741 +0000 UTC m=+0.333712253 container start 526499193ea3c0b3d7d0921b84721ef025a3efe76cd7e1a010f62d7b5c775548 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3, GIT_BRANCH=HEAD, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS) 2024-06-18T03:11:16.868 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:16 smithi163 ceph-osd[112419]: -- 172.21.15.163:0/456036806 <== mon.1 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56409565fa40 con 0x56409641e000 2024-06-18T03:11:16.868 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:16 smithi163 bash[112401]: 526499193ea3c0b3d7d0921b84721ef025a3efe76cd7e1a010f62d7b5c775548 2024-06-18T03:11:16.868 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:16 smithi163 systemd[1]: Started Ceph osd.3 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:11:17.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:16 smithi163 ceph-mon[82879]: pgmap v155: 161 pgs: 33 active+undersized, 14 peering, 19 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 68/723 objects degraded (9.405%) 2024-06-18T03:11:17.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:16 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:17.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:16 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:17.243 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:16 smithi163 ceph-mon[87697]: pgmap v155: 161 pgs: 33 active+undersized, 14 peering, 19 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 68/723 objects degraded (9.405%) 2024-06-18T03:11:17.243 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:16 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:17.243 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:16 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:17.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:16 smithi186 ceph-mon[67751]: pgmap v155: 161 pgs: 33 active+undersized, 14 peering, 19 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 68/723 objects degraded (9.405%) 2024-06-18T03:11:17.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:16 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:17.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:16 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:17.542 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:17 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3[112415]: 2024-06-18T03:11:17.242+0000 7f5f8bb7b640 -1 Falling back to public interface 2024-06-18T03:11:18.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:17 smithi163 ceph-mon[87697]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-06-18T03:11:18.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:17 smithi163 ceph-mon[82879]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-06-18T03:11:18.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:17 smithi186 ceph-mon[67751]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-06-18T03:11:19.204 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:18 smithi163 ceph-mon[87697]: pgmap v156: 161 pgs: 41 active+undersized, 25 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-06-18T03:11:19.204 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:18 smithi163 ceph-mon[82879]: pgmap v156: 161 pgs: 41 active+undersized, 25 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-06-18T03:11:19.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:18 smithi186 ceph-mon[67751]: pgmap v156: 161 pgs: 41 active+undersized, 25 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-06-18T03:11:20.292 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:19 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3[112415]: 2024-06-18T03:11:19.966+0000 7f5f8bb7b640 -1 osd.3 0 read_superblock omap replica is missing. 2024-06-18T03:11:20.293 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:20 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3[112415]: 2024-06-18T03:11:20.002+0000 7f5f8bb7b640 -1 osd.3 127 log_to_monitors true 2024-06-18T03:11:21.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:20 smithi163 ceph-mon[87697]: pgmap v157: 161 pgs: 41 active+undersized, 25 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-06-18T03:11:21.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:20 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:21.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:20 smithi163 ceph-mon[87697]: from='osd.3 [v2:172.21.15.163:6826/2473792401,v1:172.21.15.163:6827/2473792401]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-18T03:11:21.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:20 smithi163 ceph-mon[87697]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-18T03:11:21.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:20 smithi163 ceph-mon[82879]: pgmap v157: 161 pgs: 41 active+undersized, 25 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-06-18T03:11:21.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:20 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:21.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:20 smithi163 ceph-mon[82879]: from='osd.3 [v2:172.21.15.163:6826/2473792401,v1:172.21.15.163:6827/2473792401]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-18T03:11:21.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:20 smithi163 ceph-mon[82879]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-18T03:11:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:20 smithi186 ceph-mon[67751]: pgmap v157: 161 pgs: 41 active+undersized, 25 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-06-18T03:11:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:20 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:20 smithi186 ceph-mon[67751]: from='osd.3 [v2:172.21.15.163:6826/2473792401,v1:172.21.15.163:6827/2473792401]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-18T03:11:21.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:20 smithi186 ceph-mon[67751]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-18T03:11:21.793 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3[112415]: 2024-06-18T03:11:21.391+0000 7f5f7f91e640 -1 osd.3 127 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T03:11:22.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[87697]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-18T03:11:22.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[87697]: osdmap e130: 8 total, 7 up, 8 in 2024-06-18T03:11:22.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[87697]: from='osd.3 [v2:172.21.15.163:6826/2473792401,v1:172.21.15.163:6827/2473792401]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:11:22.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[87697]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:11:22.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:22.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:22.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[87697]: from='osd.3 ' entity='osd.3' 2024-06-18T03:11:22.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:22.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:22.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[87697]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 25 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:22.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[82879]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-18T03:11:22.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[82879]: osdmap e130: 8 total, 7 up, 8 in 2024-06-18T03:11:22.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[82879]: from='osd.3 [v2:172.21.15.163:6826/2473792401,v1:172.21.15.163:6827/2473792401]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:11:22.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[82879]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:11:22.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:22.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:22.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[82879]: from='osd.3 ' entity='osd.3' 2024-06-18T03:11:22.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:22.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:22.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:21 smithi163 ceph-mon[82879]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 25 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:21 smithi186 ceph-mon[67751]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-18T03:11:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:21 smithi186 ceph-mon[67751]: osdmap e130: 8 total, 7 up, 8 in 2024-06-18T03:11:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:21 smithi186 ceph-mon[67751]: from='osd.3 [v2:172.21.15.163:6826/2473792401,v1:172.21.15.163:6827/2473792401]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:11:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:21 smithi186 ceph-mon[67751]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2024-06-18T03:11:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:21 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:22.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:21 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:22.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:21 smithi186 ceph-mon[67751]: from='osd.3 ' entity='osd.3' 2024-06-18T03:11:22.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:21 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:22.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:21 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:22.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:21 smithi186 ceph-mon[67751]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 25 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:23.270 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:22 smithi163 ceph-mon[87697]: pgmap v159: 161 pgs: 41 active+undersized, 25 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-06-18T03:11:23.271 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:22 smithi163 ceph-mon[87697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:11:23.271 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:22 smithi163 ceph-mon[87697]: osd.3 [v2:172.21.15.163:6826/2473792401,v1:172.21.15.163:6827/2473792401] boot 2024-06-18T03:11:23.271 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:22 smithi163 ceph-mon[87697]: osdmap e131: 8 total, 8 up, 8 in 2024-06-18T03:11:23.271 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:22 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T03:11:23.271 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:22 smithi163 ceph-mon[82879]: pgmap v159: 161 pgs: 41 active+undersized, 25 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-06-18T03:11:23.271 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:22 smithi163 ceph-mon[82879]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:11:23.271 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:22 smithi163 ceph-mon[82879]: osd.3 [v2:172.21.15.163:6826/2473792401,v1:172.21.15.163:6827/2473792401] boot 2024-06-18T03:11:23.271 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:22 smithi163 ceph-mon[82879]: osdmap e131: 8 total, 8 up, 8 in 2024-06-18T03:11:23.271 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:22 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T03:11:23.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:22 smithi186 ceph-mon[67751]: pgmap v159: 161 pgs: 41 active+undersized, 25 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-06-18T03:11:23.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:22 smithi186 ceph-mon[67751]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:11:23.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:22 smithi186 ceph-mon[67751]: osd.3 [v2:172.21.15.163:6826/2473792401,v1:172.21.15.163:6827/2473792401] boot 2024-06-18T03:11:23.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:22 smithi186 ceph-mon[67751]: osdmap e131: 8 total, 8 up, 8 in 2024-06-18T03:11:23.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:22 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-18T03:11:24.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:11:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:11:23] "GET /metrics HTTP/1.1" 200 38186 "" "Prometheus/2.43.0" 2024-06-18T03:11:24.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:24 smithi163 ceph-mon[82879]: osdmap e132: 8 total, 8 up, 8 in 2024-06-18T03:11:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:24 smithi163 ceph-mon[82879]: pgmap v162: 161 pgs: 5 peering, 36 active+undersized, 25 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 95/723 objects degraded (13.140%) 2024-06-18T03:11:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:24 smithi163 ceph-mon[87697]: osdmap e132: 8 total, 8 up, 8 in 2024-06-18T03:11:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:24 smithi163 ceph-mon[87697]: pgmap v162: 161 pgs: 5 peering, 36 active+undersized, 25 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 95/723 objects degraded (13.140%) 2024-06-18T03:11:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:24 smithi186 ceph-mon[67751]: osdmap e132: 8 total, 8 up, 8 in 2024-06-18T03:11:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:24 smithi186 ceph-mon[67751]: pgmap v162: 161 pgs: 5 peering, 36 active+undersized, 25 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 95/723 objects degraded (13.140%) 2024-06-18T03:11:25.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:11:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:11:25.561+0000 7f10a413f640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-06-18T03:11:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:11:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: pgmap v163: 161 pgs: 5 peering, 21 active+undersized, 14 active+undersized+degraded, 121 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-06-18T03:11:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: pgmap v164: 161 pgs: 5 peering, 21 active+undersized, 14 active+undersized+degraded, 121 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-06-18T03:11:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:11:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:26.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:26.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:26.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-18T03:11:26.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-18T03:11:26.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[87697]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-06-18T03:11:26.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:26.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:26.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:26.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:11:26.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: pgmap v163: 161 pgs: 5 peering, 21 active+undersized, 14 active+undersized+degraded, 121 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-06-18T03:11:26.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: pgmap v164: 161 pgs: 5 peering, 21 active+undersized, 14 active+undersized+degraded, 121 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-06-18T03:11:26.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:26.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:11:26.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:26.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:26.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:26.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-18T03:11:26.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-18T03:11:26.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:26 smithi163 ceph-mon[82879]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-06-18T03:11:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:11:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: pgmap v163: 161 pgs: 5 peering, 21 active+undersized, 14 active+undersized+degraded, 121 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-06-18T03:11:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: pgmap v164: 161 pgs: 5 peering, 21 active+undersized, 14 active+undersized+degraded, 121 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-06-18T03:11:26.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:26.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:11:26.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:26.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:26.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:26.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-18T03:11:26.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-18T03:11:26.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:26 smithi186 ceph-mon[67751]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-06-18T03:11:27.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:27 smithi163 ceph-mon[87697]: Health check update: Degraded data redundancy: 58/723 objects degraded (8.022%), 14 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:27.694 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:27 smithi163 ceph-mon[82879]: Health check update: Degraded data redundancy: 58/723 objects degraded (8.022%), 14 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:27 smithi186 ceph-mon[67751]: Health check update: Degraded data redundancy: 58/723 objects degraded (8.022%), 14 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:28 smithi163 ceph-mon[82879]: pgmap v165: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:11:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:28 smithi163 ceph-mon[82879]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 58/723 objects degraded (8.022%), 14 pgs degraded) 2024-06-18T03:11:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:28 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:28 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:11:28.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:28 smithi163 ceph-mon[87697]: pgmap v165: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:11:28.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:28 smithi163 ceph-mon[87697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 58/723 objects degraded (8.022%), 14 pgs degraded) 2024-06-18T03:11:28.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:28 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:28.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:28 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:11:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:28 smithi186 ceph-mon[67751]: pgmap v165: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:11:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:28 smithi186 ceph-mon[67751]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 58/723 objects degraded (8.022%), 14 pgs degraded) 2024-06-18T03:11:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:28 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:28 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:11:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:30 smithi163 ceph-mon[82879]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:11:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:30 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:30.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:30 smithi163 ceph-mon[87697]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:11:30.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:30 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:30 smithi186 ceph-mon[67751]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:11:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:30 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:32.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:32 smithi163 ceph-mon[87697]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-06-18T03:11:32.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:32 smithi163 ceph-mon[82879]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-06-18T03:11:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:32 smithi186 ceph-mon[67751]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-06-18T03:11:34.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:11:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:11:33] "GET /metrics HTTP/1.1" 200 38189 "" "Prometheus/2.43.0" 2024-06-18T03:11:34.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:34 smithi163 ceph-mon[87697]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 105 B/s rd, 0 op/s 2024-06-18T03:11:34.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:34 smithi163 ceph-mon[82879]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 105 B/s rd, 0 op/s 2024-06-18T03:11:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:34 smithi186 ceph-mon[67751]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 105 B/s rd, 0 op/s 2024-06-18T03:11:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:36 smithi163 ceph-mon[82879]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-06-18T03:11:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:36 smithi163 ceph-mon[87697]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-06-18T03:11:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:36 smithi186 ceph-mon[67751]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-06-18T03:11:38.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:38 smithi163 ceph-mon[87697]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-18T03:11:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:38 smithi163 ceph-mon[82879]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-18T03:11:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:38 smithi186 ceph-mon[67751]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-18T03:11:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:39.316 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:11:39.719 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:39 smithi186 ceph-mon[67751]: from='client.44442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:39.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:39 smithi163 ceph-mon[87697]: from='client.44442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:39 smithi163 ceph-mon[82879]: from='client.44442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:39.970 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:11:39.970 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (6m) 18s ago 28m 27.1M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:11:39.970 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (5m) 4m ago 28m 76.6M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:11:39.970 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 18s ago 27m 59.1M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:11:39.970 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (7m) 4m ago 32m 438M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:11:39.970 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (17m) 18s ago 34m 502M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:11:39.970 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (4m) 18s ago 34m 52.2M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:11:39.970 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (5m) 4m ago 33m 41.4M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:11:39.970 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (3m) 18s ago 33m 44.7M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:11:39.971 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (7m) 18s ago 29m 15.9M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:11:39.971 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (7m) 4m ago 28m 15.7M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:11:39.971 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (2m) 18s ago 32m 44.8M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:11:39.971 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (2m) 18s ago 32m 44.9M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:11:39.971 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (75s) 18s ago 31m 48.0M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 05782842a9a4 2024-06-18T03:11:39.971 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (23s) 18s ago 31m 11.8M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 526499193ea3 2024-06-18T03:11:39.971 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (31m) 4m ago 31m 61.6M 3248M 17.2.0 e1d6a67b021e 832c6c891bb5 2024-06-18T03:11:39.971 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (30m) 4m ago 30m 52.3M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:11:39.971 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (30m) 4m ago 30m 52.2M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:11:39.971 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (29m) 4m ago 29m 55.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:11:39.971 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (6m) 4m ago 29m 69.4M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:11:39.971 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (27m) 18s ago 27m 81.0M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:11:39.971 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (27m) 4m ago 27m 80.6M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:11:40.363 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 4 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:11:40.364 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:11:40.365 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-06-18T03:11:40.365 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 9 2024-06-18T03:11:40.365 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:11:40.365 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:11:40.694 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:11:40.694 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:11:40.694 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:11:40.694 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-06-18T03:11:40.694 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:11:40.694 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "4/8 daemons upgraded", 2024-06-18T03:11:40.694 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading osd daemons", 2024-06-18T03:11:40.695 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:11:40.695 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:11:40.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:40 smithi163 ceph-mon[87697]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:11:40.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:40 smithi163 ceph-mon[87697]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:40.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:40 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:40.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:40 smithi163 ceph-mon[87697]: from='client.44454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:40.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:40 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/634759130' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:40.695 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:40 smithi163 ceph-mon[82879]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:11:40.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:40 smithi163 ceph-mon[82879]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:40.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:40 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:40.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:40 smithi163 ceph-mon[82879]: from='client.44454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:40.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:40 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/634759130' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:40 smithi186 ceph-mon[67751]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:11:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:40 smithi186 ceph-mon[67751]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:40 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:40 smithi186 ceph-mon[67751]: from='client.44454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:40.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:40 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/634759130' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:41.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:41 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-18T03:11:41.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:41 smithi186 ceph-mon[67751]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-18T03:11:41.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:41 smithi186 ceph-mon[67751]: Upgrade: osd.4 is safe to restart 2024-06-18T03:11:41.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:41 smithi186 ceph-mon[67751]: from='client.54433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:41.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:41 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-18T03:11:41.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:41 smithi163 ceph-mon[82879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-18T03:11:41.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:41 smithi163 ceph-mon[82879]: Upgrade: osd.4 is safe to restart 2024-06-18T03:11:41.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:41 smithi163 ceph-mon[82879]: from='client.54433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:41.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:41 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-18T03:11:41.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:41 smithi163 ceph-mon[87697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-18T03:11:41.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:41 smithi163 ceph-mon[87697]: Upgrade: osd.4 is safe to restart 2024-06-18T03:11:41.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:41 smithi163 ceph-mon[87697]: from='client.54433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:11:43.222 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:42 smithi186 ceph-mon[67751]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:43.222 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:42 smithi186 ceph-mon[67751]: Upgrade: Updating osd.4 2024-06-18T03:11:43.223 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:42 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:43.223 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:42 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-18T03:11:43.223 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:42 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:43.223 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:42 smithi186 ceph-mon[67751]: Deploying daemon osd.4 on smithi186 2024-06-18T03:11:43.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:42 smithi163 ceph-mon[87697]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:43.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:42 smithi163 ceph-mon[87697]: Upgrade: Updating osd.4 2024-06-18T03:11:43.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:42 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:43.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:42 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-18T03:11:43.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:42 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:43.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:42 smithi163 ceph-mon[87697]: Deploying daemon osd.4 on smithi186 2024-06-18T03:11:43.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:42 smithi163 ceph-mon[82879]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:11:43.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:42 smithi163 ceph-mon[82879]: Upgrade: Updating osd.4 2024-06-18T03:11:43.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:42 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:43.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:42 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-18T03:11:43.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:42 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:43.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:42 smithi163 ceph-mon[82879]: Deploying daemon osd.4 on smithi186 2024-06-18T03:11:44.229 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:11:44.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:11:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:11:43] "GET /metrics HTTP/1.1" 200 38188 "" "Prometheus/2.43.0" 2024-06-18T03:11:44.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:11:44.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:11:44.953 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:44 smithi186 systemd[1]: Stopping Ceph osd.4 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:11:45.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:44 smithi163 ceph-mon[87697]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:11:45.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:44 smithi163 ceph-mon[82879]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:11:45.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:44 smithi186 ceph-mon[67751]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:11:45.318 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:45 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[29287]: 2024-06-18T03:11:45.042+0000 7f3034a7d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:11:45.318 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:45 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[29287]: 2024-06-18T03:11:45.042+0000 7f3034a7d700 -1 osd.4 132 *** Got signal Terminated *** 2024-06-18T03:11:45.319 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:45 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[29287]: 2024-06-18T03:11:45.042+0000 7f3034a7d700 -1 osd.4 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:11:46.286 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:45 smithi186 ceph-mon[67751]: osd.4 marked itself down and dead 2024-06-18T03:11:46.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:45 smithi163 ceph-mon[82879]: osd.4 marked itself down and dead 2024-06-18T03:11:46.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:45 smithi163 ceph-mon[87697]: osd.4 marked itself down and dead 2024-06-18T03:11:46.568 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:46 smithi186 podman[78168]: 2024-06-18 03:11:46.285970918 +0000 UTC m=+1.369219312 container died 832c6c891bb545f357e9fbc95035153df89e7aea3f1c85d5102550fbeef22b1c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, distribution-scope=public, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_CLEAN=True, release=754, CEPH_POINT_RELEASE=-17.2.0, ceph=True, version=8, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container) 2024-06-18T03:11:46.911 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:46 smithi186 podman[78168]: 2024-06-18 03:11:46.684618767 +0000 UTC m=+1.767867157 container cleanup 832c6c891bb545f357e9fbc95035153df89e7aea3f1c85d5102550fbeef22b1c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.openshift.tags=base centos centos-stream, release=754, com.redhat.component=centos-stream-container, name=centos-stream, version=8, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vcs-type=git, io.k8s.display-name=CentOS Stream 8, architecture=x86_64) 2024-06-18T03:11:46.911 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:46 smithi186 bash[78168]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4 2024-06-18T03:11:47.261 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:46 smithi186 ceph-mon[67751]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:11:47.261 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:46 smithi186 ceph-mon[67751]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:11:47.261 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:46 smithi186 ceph-mon[67751]: osdmap e133: 8 total, 7 up, 8 in 2024-06-18T03:11:47.261 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:46 smithi186 podman[78194]: 2024-06-18 03:11:46.911001128 +0000 UTC m=+0.625177443 container remove 832c6c891bb545f357e9fbc95035153df89e7aea3f1c85d5102550fbeef22b1c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4, distribution-scope=public, GIT_CLEAN=True, architecture=x86_64, release=754, io.openshift.expose-services=, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, name=centos-stream, ceph=True, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux ) 2024-06-18T03:11:47.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:46 smithi163 ceph-mon[82879]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:11:47.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:46 smithi163 ceph-mon[82879]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:11:47.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:46 smithi163 ceph-mon[82879]: osdmap e133: 8 total, 7 up, 8 in 2024-06-18T03:11:47.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:46 smithi163 ceph-mon[87697]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:11:47.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:46 smithi163 ceph-mon[87697]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:11:47.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:46 smithi163 ceph-mon[87697]: osdmap e133: 8 total, 7 up, 8 in 2024-06-18T03:11:47.531 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:47 smithi186 podman[78283]: 2024-06-18 03:11:47.164277954 +0000 UTC m=+0.017597939 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:11:47.532 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:47 smithi186 podman[78283]: 2024-06-18 03:11:47.26209991 +0000 UTC m=+0.115419892 container create 09fee202458c2b600369a1e23cae39c5176cbbbd97d8f153197f1872408fedc2 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.build-date=20240612) 2024-06-18T03:11:47.532 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:47 smithi186 podman[78283]: 2024-06-18 03:11:47.439565053 +0000 UTC m=+0.292885029 container init 09fee202458c2b600369a1e23cae39c5176cbbbd97d8f153197f1872408fedc2 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0) 2024-06-18T03:11:47.532 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:47 smithi186 podman[78283]: 2024-06-18 03:11:47.442823679 +0000 UTC m=+0.296143656 container start 09fee202458c2b600369a1e23cae39c5176cbbbd97d8f153197f1872408fedc2 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, GIT_CLEAN=True) 2024-06-18T03:11:47.532 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:47 smithi186 podman[78283]: 2024-06-18 03:11:47.531222274 +0000 UTC m=+0.384542264 container attach 09fee202458c2b600369a1e23cae39c5176cbbbd97d8f153197f1872408fedc2 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-deactivate, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-06-18T03:11:47.818 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:47 smithi186 podman[78283]: 2024-06-18 03:11:47.706889837 +0000 UTC m=+0.560209819 container died 09fee202458c2b600369a1e23cae39c5176cbbbd97d8f153197f1872408fedc2 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-deactivate, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-06-18T03:11:48.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:47 smithi163 ceph-mon[87697]: osdmap e134: 8 total, 7 up, 8 in 2024-06-18T03:11:48.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:47 smithi163 ceph-mon[82879]: osdmap e134: 8 total, 7 up, 8 in 2024-06-18T03:11:48.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:47 smithi186 ceph-mon[67751]: osdmap e134: 8 total, 7 up, 8 in 2024-06-18T03:11:48.818 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:48 smithi186 podman[78305]: 2024-06-18 03:11:48.385093634 +0000 UTC m=+0.671851929 container remove 09fee202458c2b600369a1e23cae39c5176cbbbd97d8f153197f1872408fedc2 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-18T03:11:48.818 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:48 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.4.service: Deactivated successfully. 2024-06-18T03:11:48.819 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:48 smithi186 systemd[1]: Stopped Ceph osd.4 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:11:48.819 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:48 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.4.service: Consumed 16.680s CPU time. 2024-06-18T03:11:48.819 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:48 smithi186 systemd[1]: Starting Ceph osd.4 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:11:49.196 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:48 smithi186 ceph-mon[67751]: pgmap v177: 161 pgs: 3 active+undersized, 21 stale+active+clean, 6 active+undersized+degraded, 131 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 28/723 objects degraded (3.873%) 2024-06-18T03:11:49.197 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:48 smithi186 ceph-mon[67751]: Health check failed: Degraded data redundancy: 28/723 objects degraded (3.873%), 6 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:49.197 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:48 smithi186 podman[78407]: 2024-06-18 03:11:48.724957386 +0000 UTC m=+0.017474148 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:11:49.197 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:48 smithi186 podman[78407]: 2024-06-18 03:11:48.869392344 +0000 UTC m=+0.161909107 container create fd34580ebc31bd3c54e59da5c10b392590fac6978ae4407087426815e5a655bf (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True) 2024-06-18T03:11:49.197 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 podman[78407]: 2024-06-18 03:11:49.197272123 +0000 UTC m=+0.489788876 container init fd34580ebc31bd3c54e59da5c10b392590fac6978ae4407087426815e5a655bf (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-activate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240612, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-3781e3a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-06-18T03:11:49.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:48 smithi163 ceph-mon[87697]: pgmap v177: 161 pgs: 3 active+undersized, 21 stale+active+clean, 6 active+undersized+degraded, 131 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 28/723 objects degraded (3.873%) 2024-06-18T03:11:49.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:48 smithi163 ceph-mon[87697]: Health check failed: Degraded data redundancy: 28/723 objects degraded (3.873%), 6 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:49.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:48 smithi163 ceph-mon[82879]: pgmap v177: 161 pgs: 3 active+undersized, 21 stale+active+clean, 6 active+undersized+degraded, 131 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 28/723 objects degraded (3.873%) 2024-06-18T03:11:49.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:48 smithi163 ceph-mon[82879]: Health check failed: Degraded data redundancy: 28/723 objects degraded (3.873%), 6 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:49.568 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 podman[78407]: 2024-06-18 03:11:49.200816128 +0000 UTC m=+0.493332890 container start fd34580ebc31bd3c54e59da5c10b392590fac6978ae4407087426815e5a655bf (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-06-18T03:11:49.568 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 podman[78407]: 2024-06-18 03:11:49.263688956 +0000 UTC m=+0.556205736 container attach fd34580ebc31bd3c54e59da5c10b392590fac6978ae4407087426815e5a655bf (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-activate, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-06-18T03:11:49.819 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-activate[78423]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-18T03:11:49.819 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 bash[78407]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-18T03:11:49.819 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-activate[78423]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-18T03:11:49.819 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 bash[78407]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-18T03:11:49.819 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-activate[78423]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-18T03:11:49.819 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 bash[78407]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-18T03:11:49.820 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-activate[78423]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-18T03:11:49.820 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 bash[78407]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-18T03:11:49.820 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-activate[78423]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-06-18T03:11:49.820 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 bash[78407]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-06-18T03:11:49.820 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-activate[78423]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-18T03:11:49.820 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 bash[78407]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-18T03:11:49.820 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-activate[78423]: --> ceph-volume raw activate successful for osd ID: 4 2024-06-18T03:11:49.821 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 bash[78407]: --> ceph-volume raw activate successful for osd ID: 4 2024-06-18T03:11:49.821 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:49 smithi186 podman[78407]: 2024-06-18 03:11:49.621993149 +0000 UTC m=+0.914509906 container died fd34580ebc31bd3c54e59da5c10b392590fac6978ae4407087426815e5a655bf (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-activate, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-18T03:11:50.318 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:50 smithi186 podman[78541]: 2024-06-18 03:11:50.158779809 +0000 UTC m=+0.531154523 container remove fd34580ebc31bd3c54e59da5c10b392590fac6978ae4407087426815e5a655bf (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240612, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, RELEASE=squid-3781e3a, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-06-18T03:11:50.687 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:50 smithi186 podman[78602]: 2024-06-18 03:11:50.275632367 +0000 UTC m=+0.018361554 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:11:50.687 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:50 smithi186 podman[78602]: 2024-06-18 03:11:50.393003728 +0000 UTC m=+0.135732908 container create 2c0ea5619d3880176bd7e31b04f022f26452351b124c02be7514bd8ea200b737 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, io.buildah.version=1.36.0) 2024-06-18T03:11:50.687 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:50 smithi186 podman[78602]: 2024-06-18 03:11:50.595313608 +0000 UTC m=+0.338042799 container init 2c0ea5619d3880176bd7e31b04f022f26452351b124c02be7514bd8ea200b737 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS) 2024-06-18T03:11:50.687 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:50 smithi186 podman[78602]: 2024-06-18 03:11:50.598495234 +0000 UTC m=+0.341224412 container start 2c0ea5619d3880176bd7e31b04f022f26452351b124c02be7514bd8ea200b737 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3781e3a, org.label-schema.build-date=20240612, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-06-18T03:11:50.687 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:50 smithi186 ceph-osd[78621]: -- 172.21.15.186:0/1761030452 <== mon.2 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557771775a40 con 0x557772534400 2024-06-18T03:11:50.687 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:50 smithi186 bash[78602]: 2c0ea5619d3880176bd7e31b04f022f26452351b124c02be7514bd8ea200b737 2024-06-18T03:11:50.970 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:50 smithi186 systemd[1]: Started Ceph osd.4 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:11:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:50 smithi163 ceph-mon[87697]: pgmap v178: 161 pgs: 27 active+undersized, 9 stale+active+clean, 20 active+undersized+degraded, 105 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 99/723 objects degraded (13.693%) 2024-06-18T03:11:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:50 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:50 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:51.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:50 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:51.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:50 smithi163 ceph-mon[82879]: pgmap v178: 161 pgs: 27 active+undersized, 9 stale+active+clean, 20 active+undersized+degraded, 105 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 99/723 objects degraded (13.693%) 2024-06-18T03:11:51.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:50 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:51.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:50 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:51.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:50 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:51.300 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:11:51.140+0000 7f116f89a640 -1 Falling back to public interface 2024-06-18T03:11:51.300 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:50 smithi186 ceph-mon[67751]: pgmap v178: 161 pgs: 27 active+undersized, 9 stale+active+clean, 20 active+undersized+degraded, 105 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 99/723 objects degraded (13.693%) 2024-06-18T03:11:51.300 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:50 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:11:51.300 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:50 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:51.300 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:50 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:53.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:52 smithi163 ceph-mon[87697]: pgmap v179: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-06-18T03:11:53.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:52 smithi163 ceph-mon[82879]: pgmap v179: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-06-18T03:11:53.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:52 smithi186 ceph-mon[67751]: pgmap v179: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-06-18T03:11:54.247 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:53 smithi186 ceph-mon[67751]: from='osd.4 [v2:172.21.15.186:6800/549446173,v1:172.21.15.186:6801/549446173]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-18T03:11:54.247 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:53 smithi186 ceph-mon[67751]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-18T03:11:54.247 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:11:53.863+0000 7f116f89a640 -1 osd.4 0 read_superblock omap replica is missing. 2024-06-18T03:11:54.247 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:11:53.893+0000 7f116f89a640 -1 osd.4 132 log_to_monitors true 2024-06-18T03:11:54.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:11:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:11:53] "GET /metrics HTTP/1.1" 200 38188 "" "Prometheus/2.43.0" 2024-06-18T03:11:54.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:53 smithi163 ceph-mon[87697]: from='osd.4 [v2:172.21.15.186:6800/549446173,v1:172.21.15.186:6801/549446173]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-18T03:11:54.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:53 smithi163 ceph-mon[87697]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-18T03:11:54.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:53 smithi163 ceph-mon[82879]: from='osd.4 [v2:172.21.15.186:6800/549446173,v1:172.21.15.186:6801/549446173]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-18T03:11:54.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:53 smithi163 ceph-mon[82879]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-18T03:11:54.568 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:11:54 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:11:54.453+0000 7f116363d640 -1 osd.4 132 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T03:11:55.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[87697]: pgmap v180: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-06-18T03:11:55.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[87697]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-18T03:11:55.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[87697]: osdmap e135: 8 total, 7 up, 8 in 2024-06-18T03:11:55.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[87697]: from='osd.4 [v2:172.21.15.186:6800/549446173,v1:172.21.15.186:6801/549446173]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:11:55.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[87697]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:11:55.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[87697]: from='osd.4 ' entity='osd.4' 2024-06-18T03:11:55.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:55.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:55.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[82879]: pgmap v180: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-06-18T03:11:55.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[82879]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-18T03:11:55.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[82879]: osdmap e135: 8 total, 7 up, 8 in 2024-06-18T03:11:55.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[82879]: from='osd.4 [v2:172.21.15.186:6800/549446173,v1:172.21.15.186:6801/549446173]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:11:55.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[82879]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:11:55.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[82879]: from='osd.4 ' entity='osd.4' 2024-06-18T03:11:55.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:55.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:54 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:55.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:54 smithi186 ceph-mon[67751]: pgmap v180: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-06-18T03:11:55.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:54 smithi186 ceph-mon[67751]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-18T03:11:55.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:54 smithi186 ceph-mon[67751]: osdmap e135: 8 total, 7 up, 8 in 2024-06-18T03:11:55.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:54 smithi186 ceph-mon[67751]: from='osd.4 [v2:172.21.15.186:6800/549446173,v1:172.21.15.186:6801/549446173]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:11:55.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:54 smithi186 ceph-mon[67751]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:11:55.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:54 smithi186 ceph-mon[67751]: from='osd.4 ' entity='osd.4' 2024-06-18T03:11:55.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:54 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:55.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:54 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:56.174 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:55 smithi186 ceph-mon[67751]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:11:56.174 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:55 smithi186 ceph-mon[67751]: osd.4 [v2:172.21.15.186:6800/549446173,v1:172.21.15.186:6801/549446173] boot 2024-06-18T03:11:56.174 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:55 smithi186 ceph-mon[67751]: osdmap e136: 8 total, 8 up, 8 in 2024-06-18T03:11:56.174 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:55 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T03:11:56.174 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:55 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:56.174 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:55 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:55 smithi163 ceph-mon[82879]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:11:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:55 smithi163 ceph-mon[82879]: osd.4 [v2:172.21.15.186:6800/549446173,v1:172.21.15.186:6801/549446173] boot 2024-06-18T03:11:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:55 smithi163 ceph-mon[82879]: osdmap e136: 8 total, 8 up, 8 in 2024-06-18T03:11:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:55 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T03:11:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:55 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:56.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:55 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:56.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:55 smithi163 ceph-mon[87697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:11:56.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:55 smithi163 ceph-mon[87697]: osd.4 [v2:172.21.15.186:6800/549446173,v1:172.21.15.186:6801/549446173] boot 2024-06-18T03:11:56.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:55 smithi163 ceph-mon[87697]: osdmap e136: 8 total, 8 up, 8 in 2024-06-18T03:11:56.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:55 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-18T03:11:56.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:55 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:56.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:55 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:57.267 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:56 smithi186 ceph-mon[67751]: pgmap v182: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-06-18T03:11:57.268 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:56 smithi186 ceph-mon[67751]: osdmap e137: 8 total, 8 up, 8 in 2024-06-18T03:11:57.268 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:56 smithi186 ceph-mon[67751]: Health check update: Degraded data redundancy: 124/723 objects degraded (17.151%), 26 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:57.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:56 smithi163 ceph-mon[87697]: pgmap v182: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-06-18T03:11:57.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:56 smithi163 ceph-mon[87697]: osdmap e137: 8 total, 8 up, 8 in 2024-06-18T03:11:57.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:56 smithi163 ceph-mon[87697]: Health check update: Degraded data redundancy: 124/723 objects degraded (17.151%), 26 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:57.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:56 smithi163 ceph-mon[82879]: pgmap v182: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-06-18T03:11:57.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:56 smithi163 ceph-mon[82879]: osdmap e137: 8 total, 8 up, 8 in 2024-06-18T03:11:57.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:56 smithi163 ceph-mon[82879]: Health check update: Degraded data redundancy: 124/723 objects degraded (17.151%), 26 pgs degraded (PG_DEGRADED) 2024-06-18T03:11:59.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:11:58 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:11:58.939+0000 7f10a413f640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-18T03:11:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[87697]: pgmap v185: 161 pgs: 4 peering, 37 active+undersized, 20 active+undersized+degraded, 100 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 100/723 objects degraded (13.831%) 2024-06-18T03:11:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:11:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:11:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:11:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:59.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:59.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:59.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-18T03:11:59.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[82879]: pgmap v185: 161 pgs: 4 peering, 37 active+undersized, 20 active+undersized+degraded, 100 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 100/723 objects degraded (13.831%) 2024-06-18T03:11:59.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:59.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:11:59.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:59.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:59.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:59.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:11:59.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:59.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:11:59.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:59.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:59.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:59.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:11:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-18T03:11:59.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:59 smithi186 ceph-mon[67751]: pgmap v185: 161 pgs: 4 peering, 37 active+undersized, 20 active+undersized+degraded, 100 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 100/723 objects degraded (13.831%) 2024-06-18T03:11:59.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:59.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:11:59.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:59.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:59.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:11:59.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:11:59.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:11:59.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:11:59.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:59.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:59.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:11:59.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:11:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-18T03:12:00.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:00 smithi163 ceph-mon[87697]: pgmap v186: 161 pgs: 4 peering, 33 active+undersized, 20 active+undersized+degraded, 104 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 953 B/s rd, 0 op/s; 100/723 objects degraded (13.831%) 2024-06-18T03:12:00.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:00 smithi163 ceph-mon[87697]: pgmap v187: 161 pgs: 4 peering, 33 active+undersized, 20 active+undersized+degraded, 104 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 100/723 objects degraded (13.831%) 2024-06-18T03:12:00.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:00 smithi163 ceph-mon[87697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-18T03:12:00.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:00 smithi163 ceph-mon[87697]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-18T03:12:00.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:00 smithi163 ceph-mon[82879]: pgmap v186: 161 pgs: 4 peering, 33 active+undersized, 20 active+undersized+degraded, 104 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 953 B/s rd, 0 op/s; 100/723 objects degraded (13.831%) 2024-06-18T03:12:00.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:00 smithi163 ceph-mon[82879]: pgmap v187: 161 pgs: 4 peering, 33 active+undersized, 20 active+undersized+degraded, 104 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 100/723 objects degraded (13.831%) 2024-06-18T03:12:00.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:00 smithi163 ceph-mon[82879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-18T03:12:00.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:00 smithi163 ceph-mon[82879]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-18T03:12:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:00 smithi186 ceph-mon[67751]: pgmap v186: 161 pgs: 4 peering, 33 active+undersized, 20 active+undersized+degraded, 104 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 953 B/s rd, 0 op/s; 100/723 objects degraded (13.831%) 2024-06-18T03:12:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:00 smithi186 ceph-mon[67751]: pgmap v187: 161 pgs: 4 peering, 33 active+undersized, 20 active+undersized+degraded, 104 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 100/723 objects degraded (13.831%) 2024-06-18T03:12:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:00 smithi186 ceph-mon[67751]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-18T03:12:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:00 smithi186 ceph-mon[67751]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-18T03:12:01.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:01 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:01 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:01.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:01 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:02.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:02 smithi163 ceph-mon[87697]: pgmap v188: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 953 B/s rd, 0 op/s 2024-06-18T03:12:02.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:02 smithi163 ceph-mon[87697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 100/723 objects degraded (13.831%), 20 pgs degraded) 2024-06-18T03:12:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:02 smithi163 ceph-mon[82879]: pgmap v188: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 953 B/s rd, 0 op/s 2024-06-18T03:12:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:02 smithi163 ceph-mon[82879]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 100/723 objects degraded (13.831%), 20 pgs degraded) 2024-06-18T03:12:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:02 smithi186 ceph-mon[67751]: pgmap v188: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 953 B/s rd, 0 op/s 2024-06-18T03:12:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:02 smithi186 ceph-mon[67751]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 100/723 objects degraded (13.831%), 20 pgs degraded) 2024-06-18T03:12:04.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:12:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:12:03] "GET /metrics HTTP/1.1" 200 38061 "" "Prometheus/2.43.0" 2024-06-18T03:12:04.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:04 smithi163 ceph-mon[87697]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:12:04.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:04 smithi163 ceph-mon[82879]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:12:04.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:04 smithi186 ceph-mon[67751]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:12:06.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:06 smithi163 ceph-mon[87697]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-06-18T03:12:06.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:06 smithi163 ceph-mon[82879]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-06-18T03:12:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:06 smithi186 ceph-mon[67751]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-06-18T03:12:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:08 smithi163 ceph-mon[87697]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:12:08.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:08 smithi163 ceph-mon[82879]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:12:08.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:08 smithi186 ceph-mon[67751]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:12:09.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:09 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:09.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:09 smithi186 ceph-mon[67751]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:12:09.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:09 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:09.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:09 smithi163 ceph-mon[87697]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:12:09.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:09 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:09.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:09 smithi163 ceph-mon[82879]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:12:10.732 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:10 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:10.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:10 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:10.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:10 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:11.051 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:12:11.700 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:12:11.700 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (6m) 50s ago 28m 27.1M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:12:11.700 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (6m) 16s ago 28m 80.1M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:12:11.700 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 50s ago 27m 59.1M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:12:11.700 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (7m) 16s ago 33m 443M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:12:11.700 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (17m) 50s ago 35m 502M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:12:11.700 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (4m) 50s ago 35m 52.2M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:12:11.700 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (5m) 16s ago 33m 48.0M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:12:11.700 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (4m) 50s ago 33m 44.7M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:12:11.701 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (7m) 50s ago 29m 15.9M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:12:11.701 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (7m) 16s ago 29m 15.7M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:12:11.701 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (3m) 50s ago 33m 44.8M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:12:11.701 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (2m) 50s ago 32m 44.9M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:12:11.701 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (107s) 50s ago 32m 48.0M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 05782842a9a4 2024-06-18T03:12:11.701 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (55s) 50s ago 31m 11.8M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 526499193ea3 2024-06-18T03:12:11.701 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (21s) 16s ago 31m 12.1M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 2c0ea5619d38 2024-06-18T03:12:11.701 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (31m) 16s ago 31m 57.2M 3248M 17.2.0 e1d6a67b021e f285094f7e76 2024-06-18T03:12:11.701 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (30m) 16s ago 30m 55.5M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:12:11.701 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (30m) 16s ago 30m 59.2M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:12:11.701 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (7m) 16s ago 29m 69.6M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:12:11.701 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (28m) 50s ago 28m 81.0M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:12:11.701 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (28m) 16s ago 28m 81.9M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:12:11.702 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:11 smithi163 ceph-mon[87697]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:11.702 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:11 smithi163 ceph-mon[87697]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:11.702 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:11 smithi163 ceph-mon[87697]: from='client.44484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:11.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:11 smithi163 ceph-mon[82879]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:11.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:11 smithi163 ceph-mon[82879]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:11.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:11 smithi163 ceph-mon[82879]: from='client.44484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:11.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:11 smithi186 ceph-mon[67751]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:11.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:11 smithi186 ceph-mon[67751]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:11.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:11 smithi186 ceph-mon[67751]: from='client.44484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:12.093 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:12:12.093 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:12:12.094 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:12:12.094 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:12:12.094 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:12:12.094 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:12:12.094 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:12:12.094 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:12:12.094 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-06-18T03:12:12.094 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 5 2024-06-18T03:12:12.094 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:12:12.094 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:12:12.094 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:12:12.094 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:12:12.094 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:12:12.095 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-06-18T03:12:12.095 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 10 2024-06-18T03:12:12.095 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:12:12.095 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:12:12.435 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:12:12.435 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:12:12.435 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:12:12.435 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-06-18T03:12:12.435 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:12:12.435 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "5/8 daemons upgraded", 2024-06-18T03:12:12.435 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading osd daemons", 2024-06-18T03:12:12.435 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:12:12.435 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:12:12.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:12 smithi163 ceph-mon[87697]: from='client.54454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:12.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:12 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/3015515964' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:12.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:12 smithi163 ceph-mon[82879]: from='client.54454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:12.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:12 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/3015515964' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:12 smithi186 ceph-mon[67751]: from='client.54454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:12.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:12 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/3015515964' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:14.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:12:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:12:13] "GET /metrics HTTP/1.1" 200 38195 "" "Prometheus/2.43.0" 2024-06-18T03:12:14.599 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:14 smithi186 ceph-mon[67751]: from='client.44502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:14.599 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:14 smithi186 ceph-mon[67751]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:14.599 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:14.599 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:12:14.599 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-18T03:12:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:14 smithi163 ceph-mon[82879]: from='client.44502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:14 smithi163 ceph-mon[82879]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:12:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-18T03:12:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:14 smithi163 ceph-mon[87697]: from='client.44502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:14 smithi163 ceph-mon[87697]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:14.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:14.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:12:14.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-18T03:12:15.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:15 smithi186 ceph-mon[67751]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-18T03:12:15.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:15 smithi186 ceph-mon[67751]: Upgrade: osd.5 is safe to restart 2024-06-18T03:12:15.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:15 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:15.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:15 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-18T03:12:15.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:15 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:12:15.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:15 smithi163 ceph-mon[87697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-18T03:12:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:15 smithi163 ceph-mon[87697]: Upgrade: osd.5 is safe to restart 2024-06-18T03:12:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:15 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:15 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-18T03:12:15.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:15 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:12:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:15 smithi163 ceph-mon[82879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-18T03:12:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:15 smithi163 ceph-mon[82879]: Upgrade: osd.5 is safe to restart 2024-06-18T03:12:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:15 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:15 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-18T03:12:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:15 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:12:16.696 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:16 smithi186 ceph-mon[67751]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:16.696 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:16 smithi186 ceph-mon[67751]: Upgrade: Updating osd.5 2024-06-18T03:12:16.696 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:16 smithi186 ceph-mon[67751]: Deploying daemon osd.5 on smithi186 2024-06-18T03:12:16.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:16 smithi163 ceph-mon[87697]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:16.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:16 smithi163 ceph-mon[87697]: Upgrade: Updating osd.5 2024-06-18T03:12:16.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:16 smithi163 ceph-mon[87697]: Deploying daemon osd.5 on smithi186 2024-06-18T03:12:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:16 smithi163 ceph-mon[82879]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:16 smithi163 ceph-mon[82879]: Upgrade: Updating osd.5 2024-06-18T03:12:16.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:16 smithi163 ceph-mon[82879]: Deploying daemon osd.5 on smithi186 2024-06-18T03:12:18.457 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:18 smithi186 ceph-mon[67751]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:18.457 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:18 smithi186 systemd[1]: Stopping Ceph osd.5 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:12:18.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:18 smithi163 ceph-mon[87697]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:18.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:18 smithi163 ceph-mon[82879]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:18.818 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[32210]: 2024-06-18T03:12:18.455+0000 7f78701a5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:12:18.818 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[32210]: 2024-06-18T03:12:18.455+0000 7f78701a5700 -1 osd.5 137 *** Got signal Terminated *** 2024-06-18T03:12:18.818 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[32210]: 2024-06-18T03:12:18.455+0000 7f78701a5700 -1 osd.5 137 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:12:19.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:19 smithi163 ceph-mon[87697]: osd.5 marked itself down and dead 2024-06-18T03:12:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:19 smithi163 ceph-mon[82879]: osd.5 marked itself down and dead 2024-06-18T03:12:19.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:19 smithi186 ceph-mon[67751]: osd.5 marked itself down and dead 2024-06-18T03:12:20.131 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:19 smithi186 podman[83102]: 2024-06-18 03:12:19.863801353 +0000 UTC m=+1.519721835 container died f285094f7e76c5a7a64d91b9ae2df264c991213ae3830ca6269601dac9794028 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.buildah.version=1.19.8, ceph=True, distribution-scope=public, RELEASE=HEAD, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, vcs-type=git, maintainer=Guillaume Abrioux , architecture=x86_64, io.openshift.tags=base centos centos-stream) 2024-06-18T03:12:20.523 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:20 smithi186 ceph-mon[67751]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:20.523 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:20 smithi186 ceph-mon[67751]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:12:20.523 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:20 smithi186 ceph-mon[67751]: osdmap e138: 8 total, 7 up, 8 in 2024-06-18T03:12:20.524 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:20 smithi186 podman[83102]: 2024-06-18 03:12:20.24712457 +0000 UTC m=+1.903045041 container cleanup f285094f7e76c5a7a64d91b9ae2df264c991213ae3830ca6269601dac9794028 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, release=754, RELEASE=HEAD, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.buildah.version=1.19.8, name=centos-stream, vendor=Red Hat, Inc.) 2024-06-18T03:12:20.524 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:20 smithi186 bash[83102]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5 2024-06-18T03:12:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:20 smithi163 ceph-mon[82879]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:20 smithi163 ceph-mon[82879]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:12:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:20 smithi163 ceph-mon[82879]: osdmap e138: 8 total, 7 up, 8 in 2024-06-18T03:12:20.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:20 smithi163 ceph-mon[87697]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:20.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:20 smithi163 ceph-mon[87697]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:12:20.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:20 smithi163 ceph-mon[87697]: osdmap e138: 8 total, 7 up, 8 in 2024-06-18T03:12:20.818 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:20 smithi186 podman[83118]: 2024-06-18 03:12:20.523626458 +0000 UTC m=+0.651396530 container remove f285094f7e76c5a7a64d91b9ae2df264c991213ae3830ca6269601dac9794028 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, name=centos-stream, vendor=Red Hat, Inc., version=8, release=754, architecture=x86_64, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-06-18T03:12:21.118 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:20 smithi186 podman[83209]: 2024-06-18 03:12:20.74777811 +0000 UTC m=+0.017453869 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:12:21.118 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:20 smithi186 podman[83209]: 2024-06-18 03:12:20.891261496 +0000 UTC m=+0.160937248 container create d12dcf71e46155f955570cc92f743809506b4a8f2fa113c6508422ab743e52a0 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a, GIT_CLEAN=True) 2024-06-18T03:12:21.118 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:21 smithi186 podman[83209]: 2024-06-18 03:12:21.118456677 +0000 UTC m=+0.388132432 container init d12dcf71e46155f955570cc92f743809506b4a8f2fa113c6508422ab743e52a0 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-deactivate, io.buildah.version=1.36.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-06-18T03:12:21.371 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:21 smithi186 podman[83209]: 2024-06-18 03:12:21.121840982 +0000 UTC m=+0.391516734 container start d12dcf71e46155f955570cc92f743809506b4a8f2fa113c6508422ab743e52a0 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-deactivate, RELEASE=squid-3781e3a, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240612) 2024-06-18T03:12:21.371 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:21 smithi186 podman[83209]: 2024-06-18 03:12:21.201828864 +0000 UTC m=+0.471504627 container attach d12dcf71e46155f955570cc92f743809506b4a8f2fa113c6508422ab743e52a0 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-deactivate, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612) 2024-06-18T03:12:21.697 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:21 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:21.698 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:21 smithi186 ceph-mon[67751]: osdmap e139: 8 total, 7 up, 8 in 2024-06-18T03:12:21.698 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:21 smithi186 podman[83209]: 2024-06-18 03:12:21.3867107 +0000 UTC m=+0.656386451 container died d12dcf71e46155f955570cc92f743809506b4a8f2fa113c6508422ab743e52a0 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-06-18T03:12:21.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:21 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:21.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:21 smithi163 ceph-mon[87697]: osdmap e139: 8 total, 7 up, 8 in 2024-06-18T03:12:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:21 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:21 smithi163 ceph-mon[82879]: osdmap e139: 8 total, 7 up, 8 in 2024-06-18T03:12:22.319 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:22 smithi186 podman[83232]: 2024-06-18 03:12:22.005703489 +0000 UTC m=+0.612118028 container remove d12dcf71e46155f955570cc92f743809506b4a8f2fa113c6508422ab743e52a0 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-3781e3a, GIT_CLEAN=True) 2024-06-18T03:12:22.319 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:22 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.5.service: Deactivated successfully. 2024-06-18T03:12:22.319 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:22 smithi186 systemd[1]: Stopped Ceph osd.5 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:12:22.319 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:22 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.5.service: Consumed 15.689s CPU time. 2024-06-18T03:12:22.319 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:22 smithi186 systemd[1]: Starting Ceph osd.5 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:12:22.717 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:22 smithi186 ceph-mon[67751]: pgmap v200: 161 pgs: 11 active+undersized, 16 stale+active+clean, 2 active+undersized+degraded, 132 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 12/723 objects degraded (1.660%) 2024-06-18T03:12:22.718 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:22 smithi186 ceph-mon[67751]: Health check failed: Degraded data redundancy: 12/723 objects degraded (1.660%), 2 pgs degraded (PG_DEGRADED) 2024-06-18T03:12:22.718 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:22 smithi186 podman[83334]: 2024-06-18 03:12:22.338723031 +0000 UTC m=+0.018086338 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:12:22.718 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:22 smithi186 podman[83334]: 2024-06-18 03:12:22.46521006 +0000 UTC m=+0.144573368 container create 08171fab875a5460769516a759d426e0e6d1f4520b2503871ac5eb9bb9c1c978 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-activate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240612) 2024-06-18T03:12:22.718 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:22 smithi186 podman[83334]: 2024-06-18 03:12:22.717999281 +0000 UTC m=+0.397362598 container init 08171fab875a5460769516a759d426e0e6d1f4520b2503871ac5eb9bb9c1c978 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a) 2024-06-18T03:12:22.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:22 smithi163 ceph-mon[87697]: pgmap v200: 161 pgs: 11 active+undersized, 16 stale+active+clean, 2 active+undersized+degraded, 132 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 12/723 objects degraded (1.660%) 2024-06-18T03:12:22.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:22 smithi163 ceph-mon[87697]: Health check failed: Degraded data redundancy: 12/723 objects degraded (1.660%), 2 pgs degraded (PG_DEGRADED) 2024-06-18T03:12:22.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:22 smithi163 ceph-mon[82879]: pgmap v200: 161 pgs: 11 active+undersized, 16 stale+active+clean, 2 active+undersized+degraded, 132 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 12/723 objects degraded (1.660%) 2024-06-18T03:12:22.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:22 smithi163 ceph-mon[82879]: Health check failed: Degraded data redundancy: 12/723 objects degraded (1.660%), 2 pgs degraded (PG_DEGRADED) 2024-06-18T03:12:23.068 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:22 smithi186 podman[83334]: 2024-06-18 03:12:22.721568607 +0000 UTC m=+0.400931916 container start 08171fab875a5460769516a759d426e0e6d1f4520b2503871ac5eb9bb9c1c978 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0) 2024-06-18T03:12:23.068 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:22 smithi186 podman[83334]: 2024-06-18 03:12:22.934396017 +0000 UTC m=+0.613759339 container attach 08171fab875a5460769516a759d426e0e6d1f4520b2503871ac5eb9bb9c1c978 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a) 2024-06-18T03:12:23.424 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-activate[83348]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-18T03:12:23.424 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 bash[83334]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-18T03:12:23.424 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-activate[83348]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-18T03:12:23.424 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 bash[83334]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-18T03:12:23.425 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-activate[83348]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-18T03:12:23.425 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 bash[83334]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-18T03:12:23.425 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-activate[83348]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-18T03:12:23.425 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 bash[83334]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-18T03:12:23.425 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-activate[83348]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-06-18T03:12:23.425 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 bash[83334]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-06-18T03:12:23.425 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-activate[83348]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-18T03:12:23.425 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 bash[83334]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-18T03:12:23.425 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-activate[83348]: --> ceph-volume raw activate successful for osd ID: 5 2024-06-18T03:12:23.425 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 bash[83334]: --> ceph-volume raw activate successful for osd ID: 5 2024-06-18T03:12:23.425 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 podman[83334]: 2024-06-18 03:12:23.13623806 +0000 UTC m=+0.815601369 container died 08171fab875a5460769516a759d426e0e6d1f4520b2503871ac5eb9bb9c1c978 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-activate, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240612, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-06-18T03:12:23.713 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:23 smithi186 ceph-mon[67751]: pgmap v201: 161 pgs: 19 active+undersized, 10 stale+active+clean, 10 active+undersized+degraded, 122 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-06-18T03:12:23.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:23 smithi163 ceph-mon[87697]: pgmap v201: 161 pgs: 19 active+undersized, 10 stale+active+clean, 10 active+undersized+degraded, 122 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-06-18T03:12:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:23 smithi163 ceph-mon[82879]: pgmap v201: 161 pgs: 19 active+undersized, 10 stale+active+clean, 10 active+undersized+degraded, 122 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-06-18T03:12:24.060 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 podman[83462]: 2024-06-18 03:12:23.713220141 +0000 UTC m=+0.571164922 container remove 08171fab875a5460769516a759d426e0e6d1f4520b2503871ac5eb9bb9c1c978 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-activate, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-3781e3a, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-06-18T03:12:24.060 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 podman[83525]: 2024-06-18 03:12:23.82696154 +0000 UTC m=+0.017382641 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:12:24.060 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:23 smithi186 podman[83525]: 2024-06-18 03:12:23.939235185 +0000 UTC m=+0.129656286 container create e1d8886de2c4d14df083a15d31794138f4be51ebacfd845221d0d996e57647d1 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5, io.buildah.version=1.36.0, RELEASE=squid-3781e3a, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-06-18T03:12:24.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:12:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:12:23] "GET /metrics HTTP/1.1" 200 38195 "" "Prometheus/2.43.0" 2024-06-18T03:12:24.568 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:24 smithi186 podman[83525]: 2024-06-18 03:12:24.175103182 +0000 UTC m=+0.365524301 container init e1d8886de2c4d14df083a15d31794138f4be51ebacfd845221d0d996e57647d1 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240612, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:12:24.568 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:24 smithi186 podman[83525]: 2024-06-18 03:12:24.178441211 +0000 UTC m=+0.368862314 container start e1d8886de2c4d14df083a15d31794138f4be51ebacfd845221d0d996e57647d1 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-3781e3a) 2024-06-18T03:12:24.568 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:24 smithi186 ceph-osd[83544]: -- 172.21.15.186:0/2535849642 <== mon.2 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55af68005a40 con 0x55af68dc4400 2024-06-18T03:12:24.568 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:24 smithi186 bash[83525]: e1d8886de2c4d14df083a15d31794138f4be51ebacfd845221d0d996e57647d1 2024-06-18T03:12:24.569 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:24 smithi186 systemd[1]: Started Ceph osd.5 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:12:25.066 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:24 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:12:24.726+0000 7fa5b77d1640 -1 Falling back to public interface 2024-06-18T03:12:25.773 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:25.773 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:25.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:25.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:25.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:25.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:26.687 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:26 smithi186 ceph-mon[67751]: pgmap v202: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-06-18T03:12:26.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:26 smithi163 ceph-mon[82879]: pgmap v202: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-06-18T03:12:26.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:26 smithi163 ceph-mon[87697]: pgmap v202: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-06-18T03:12:27.530 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:27 smithi186 ceph-mon[67751]: Health check update: Degraded data redundancy: 82/723 objects degraded (11.342%), 22 pgs degraded (PG_DEGRADED) 2024-06-18T03:12:27.530 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:27 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:12:27.444+0000 7fa5b77d1640 -1 osd.5 0 read_superblock omap replica is missing. 2024-06-18T03:12:27.530 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:27 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:12:27.474+0000 7fa5b77d1640 -1 osd.5 137 log_to_monitors true 2024-06-18T03:12:27.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:27 smithi163 ceph-mon[87697]: Health check update: Degraded data redundancy: 82/723 objects degraded (11.342%), 22 pgs degraded (PG_DEGRADED) 2024-06-18T03:12:27.694 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:27 smithi163 ceph-mon[82879]: Health check update: Degraded data redundancy: 82/723 objects degraded (11.342%), 22 pgs degraded (PG_DEGRADED) 2024-06-18T03:12:28.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:28 smithi163 ceph-mon[87697]: pgmap v203: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-06-18T03:12:28.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:28 smithi163 ceph-mon[87697]: from='osd.5 [v2:172.21.15.186:6808/1105425152,v1:172.21.15.186:6809/1105425152]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-18T03:12:28.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:28 smithi163 ceph-mon[87697]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-18T03:12:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:28 smithi163 ceph-mon[82879]: pgmap v203: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-06-18T03:12:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:28 smithi163 ceph-mon[82879]: from='osd.5 [v2:172.21.15.186:6808/1105425152,v1:172.21.15.186:6809/1105425152]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-18T03:12:28.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:28 smithi163 ceph-mon[82879]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-18T03:12:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:28 smithi186 ceph-mon[67751]: pgmap v203: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-06-18T03:12:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:28 smithi186 ceph-mon[67751]: from='osd.5 [v2:172.21.15.186:6808/1105425152,v1:172.21.15.186:6809/1105425152]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-18T03:12:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:28 smithi186 ceph-mon[67751]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-18T03:12:29.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:12:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:12:28.829+0000 7fa5ab574640 -1 osd.5 137 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T03:12:29.602 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:29 smithi186 ceph-mon[67751]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-18T03:12:29.602 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:29 smithi186 ceph-mon[67751]: osdmap e140: 8 total, 7 up, 8 in 2024-06-18T03:12:29.602 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:29 smithi186 ceph-mon[67751]: from='osd.5 [v2:172.21.15.186:6808/1105425152,v1:172.21.15.186:6809/1105425152]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:12:29.602 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:29 smithi186 ceph-mon[67751]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:12:29.602 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.602 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:12:29.603 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.603 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.603 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:29 smithi186 ceph-mon[67751]: from='osd.5 ' entity='osd.5' 2024-06-18T03:12:29.603 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.603 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[87697]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-18T03:12:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[87697]: osdmap e140: 8 total, 7 up, 8 in 2024-06-18T03:12:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[87697]: from='osd.5 [v2:172.21.15.186:6808/1105425152,v1:172.21.15.186:6809/1105425152]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:12:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[87697]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:12:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:12:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[87697]: from='osd.5 ' entity='osd.5' 2024-06-18T03:12:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[82879]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-18T03:12:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[82879]: osdmap e140: 8 total, 7 up, 8 in 2024-06-18T03:12:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[82879]: from='osd.5 [v2:172.21.15.186:6808/1105425152,v1:172.21.15.186:6809/1105425152]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:12:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[82879]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:12:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:12:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[82879]: from='osd.5 ' entity='osd.5' 2024-06-18T03:12:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:29.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:30.605 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:30 smithi186 ceph-mon[67751]: pgmap v205: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-06-18T03:12:30.605 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:30 smithi186 ceph-mon[67751]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:12:30.605 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:30 smithi186 ceph-mon[67751]: osd.5 [v2:172.21.15.186:6808/1105425152,v1:172.21.15.186:6809/1105425152] boot 2024-06-18T03:12:30.605 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:30 smithi186 ceph-mon[67751]: osdmap e141: 8 total, 8 up, 8 in 2024-06-18T03:12:30.605 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:30 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T03:12:30.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:30 smithi163 ceph-mon[87697]: pgmap v205: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-06-18T03:12:30.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:30 smithi163 ceph-mon[87697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:12:30.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:30 smithi163 ceph-mon[87697]: osd.5 [v2:172.21.15.186:6808/1105425152,v1:172.21.15.186:6809/1105425152] boot 2024-06-18T03:12:30.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:30 smithi163 ceph-mon[87697]: osdmap e141: 8 total, 8 up, 8 in 2024-06-18T03:12:30.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:30 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T03:12:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:30 smithi163 ceph-mon[82879]: pgmap v205: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-06-18T03:12:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:30 smithi163 ceph-mon[82879]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:12:30.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:30 smithi163 ceph-mon[82879]: osd.5 [v2:172.21.15.186:6808/1105425152,v1:172.21.15.186:6809/1105425152] boot 2024-06-18T03:12:30.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:30 smithi163 ceph-mon[82879]: osdmap e141: 8 total, 8 up, 8 in 2024-06-18T03:12:30.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:30 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-18T03:12:31.665 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:31 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:31.665 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:31 smithi186 ceph-mon[67751]: osdmap e142: 8 total, 8 up, 8 in 2024-06-18T03:12:31.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:31 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:31.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:31 smithi163 ceph-mon[82879]: osdmap e142: 8 total, 8 up, 8 in 2024-06-18T03:12:31.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:31 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:31.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:31 smithi163 ceph-mon[87697]: osdmap e142: 8 total, 8 up, 8 in 2024-06-18T03:12:32.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:32 smithi186 ceph-mon[67751]: pgmap v208: 161 pgs: 8 peering, 31 active+undersized, 20 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 70/723 objects degraded (9.682%) 2024-06-18T03:12:32.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:32 smithi186 ceph-mon[67751]: Health check update: Degraded data redundancy: 70/723 objects degraded (9.682%), 20 pgs degraded (PG_DEGRADED) 2024-06-18T03:12:32.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:32 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:32.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:32 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:32.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:32 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:12:32.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:32 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:12:32.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:32 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:32.792 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:12:32.424+0000 7f10a413f640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-06-18T03:12:32.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[87697]: pgmap v208: 161 pgs: 8 peering, 31 active+undersized, 20 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 70/723 objects degraded (9.682%) 2024-06-18T03:12:32.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[87697]: Health check update: Degraded data redundancy: 70/723 objects degraded (9.682%), 20 pgs degraded (PG_DEGRADED) 2024-06-18T03:12:32.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:32.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:32.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:12:32.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:12:32.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:32.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[82879]: pgmap v208: 161 pgs: 8 peering, 31 active+undersized, 20 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 70/723 objects degraded (9.682%) 2024-06-18T03:12:32.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[82879]: Health check update: Degraded data redundancy: 70/723 objects degraded (9.682%), 20 pgs degraded (PG_DEGRADED) 2024-06-18T03:12:32.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:32.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:32.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:12:32.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:12:32.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:32 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:33.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[87697]: pgmap v209: 161 pgs: 8 peering, 26 active+undersized, 15 active+undersized+degraded, 112 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 748 B/s rd, 0 op/s; 52/723 objects degraded (7.192%) 2024-06-18T03:12:33.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[87697]: pgmap v210: 161 pgs: 8 peering, 26 active+undersized, 15 active+undersized+degraded, 112 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 52/723 objects degraded (7.192%) 2024-06-18T03:12:33.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:12:33.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:33.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:33.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:33.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-18T03:12:33.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[82879]: pgmap v209: 161 pgs: 8 peering, 26 active+undersized, 15 active+undersized+degraded, 112 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 748 B/s rd, 0 op/s; 52/723 objects degraded (7.192%) 2024-06-18T03:12:33.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[82879]: pgmap v210: 161 pgs: 8 peering, 26 active+undersized, 15 active+undersized+degraded, 112 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 52/723 objects degraded (7.192%) 2024-06-18T03:12:33.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:12:33.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:33.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:33.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:33.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-18T03:12:33.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:33 smithi186 ceph-mon[67751]: pgmap v209: 161 pgs: 8 peering, 26 active+undersized, 15 active+undersized+degraded, 112 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 748 B/s rd, 0 op/s; 52/723 objects degraded (7.192%) 2024-06-18T03:12:33.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:33 smithi186 ceph-mon[67751]: pgmap v210: 161 pgs: 8 peering, 26 active+undersized, 15 active+undersized+degraded, 112 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 52/723 objects degraded (7.192%) 2024-06-18T03:12:33.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:33 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:12:33.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:33 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:33.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:33 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:33.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:33 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:33.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:33 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-18T03:12:34.293 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:12:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:12:33] "GET /metrics HTTP/1.1" 200 38202 "" "Prometheus/2.43.0" 2024-06-18T03:12:34.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:34 smithi163 ceph-mon[87697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-18T03:12:34.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:34 smithi163 ceph-mon[87697]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-06-18T03:12:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:34 smithi163 ceph-mon[82879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-18T03:12:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:34 smithi163 ceph-mon[82879]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-06-18T03:12:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:34 smithi186 ceph-mon[67751]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-18T03:12:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:34 smithi186 ceph-mon[67751]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-06-18T03:12:35.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:35 smithi163 ceph-mon[87697]: pgmap v211: 161 pgs: 8 peering, 4 active+undersized, 5 active+undersized+degraded, 144 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 748 B/s rd, 0 op/s; 14/723 objects degraded (1.936%) 2024-06-18T03:12:35.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:35 smithi163 ceph-mon[82879]: pgmap v211: 161 pgs: 8 peering, 4 active+undersized, 5 active+undersized+degraded, 144 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 748 B/s rd, 0 op/s; 14/723 objects degraded (1.936%) 2024-06-18T03:12:35.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:35 smithi186 ceph-mon[67751]: pgmap v211: 161 pgs: 8 peering, 4 active+undersized, 5 active+undersized+degraded, 144 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 748 B/s rd, 0 op/s; 14/723 objects degraded (1.936%) 2024-06-18T03:12:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:36 smithi163 ceph-mon[82879]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:12:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:36 smithi163 ceph-mon[82879]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/723 objects degraded (1.936%), 5 pgs degraded) 2024-06-18T03:12:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:36 smithi163 ceph-mon[87697]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:12:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:36 smithi163 ceph-mon[87697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/723 objects degraded (1.936%), 5 pgs degraded) 2024-06-18T03:12:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:36 smithi186 ceph-mon[67751]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:12:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:36 smithi186 ceph-mon[67751]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/723 objects degraded (1.936%), 5 pgs degraded) 2024-06-18T03:12:39.772 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:39 smithi186 ceph-mon[67751]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:39.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:39 smithi163 ceph-mon[82879]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:39.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:39 smithi163 ceph-mon[87697]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:41.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:41 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:41.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:41 smithi163 ceph-mon[87697]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 973 B/s rd, 0 op/s 2024-06-18T03:12:41.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:41 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:41.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:41 smithi163 ceph-mon[82879]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 973 B/s rd, 0 op/s 2024-06-18T03:12:41.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:41 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:41.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:41 smithi186 ceph-mon[67751]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 973 B/s rd, 0 op/s 2024-06-18T03:12:42.790 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:12:43.444 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:12:43.444 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (7m) 82s ago 29m 27.1M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:12:43.444 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (7m) 14s ago 29m 80.2M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:12:43.444 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 82s ago 28m 59.1M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:12:43.445 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (8m) 14s ago 33m 443M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:12:43.445 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (18m) 82s ago 35m 502M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:12:43.445 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (5m) 82s ago 35m 52.2M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:12:43.445 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (6m) 14s ago 34m 49.2M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:12:43.445 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (5m) 82s ago 34m 44.7M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:12:43.445 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (8m) 82s ago 30m 15.9M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:12:43.445 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (8m) 14s ago 30m 15.9M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:12:43.446 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (3m) 82s ago 33m 44.8M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:12:43.446 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (3m) 82s ago 33m 44.9M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:12:43.446 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (2m) 82s ago 32m 48.0M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 05782842a9a4 2024-06-18T03:12:43.446 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (87s) 82s ago 32m 11.8M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 526499193ea3 2024-06-18T03:12:43.446 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (53s) 14s ago 32m 52.7M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 2c0ea5619d38 2024-06-18T03:12:43.446 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (19s) 14s ago 31m 14.4M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 e1d8886de2c4 2024-06-18T03:12:43.446 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (31m) 14s ago 31m 56.3M 3248M 17.2.0 e1d6a67b021e 0fcc02594571 2024-06-18T03:12:43.446 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (30m) 14s ago 30m 59.8M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:12:43.446 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (7m) 14s ago 30m 71.7M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:12:43.446 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (28m) 82s ago 28m 81.0M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:12:43.446 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (28m) 14s ago 28m 82.1M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:12:43.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:43 smithi163 ceph-mon[87697]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:12:43.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:43.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:12:43.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:43 smithi163 ceph-mon[82879]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:12:43.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:43.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:12:43.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:43 smithi186 ceph-mon[67751]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:12:43.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:43.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:12:43.833 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:12:43.833 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:12:43.833 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:12:43.833 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:12:43.833 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:12:43.834 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:12:43.834 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:12:43.834 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:12:43.834 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-06-18T03:12:43.834 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 6 2024-06-18T03:12:43.834 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:12:43.834 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:12:43.834 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:12:43.834 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:12:43.834 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:12:43.834 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-06-18T03:12:43.835 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 11 2024-06-18T03:12:43.835 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:12:43.835 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:12:44.159 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:12:44.159 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:12:44.159 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:12:44.159 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-06-18T03:12:44.159 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:12:44.159 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "6/8 daemons upgraded", 2024-06-18T03:12:44.159 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading osd daemons", 2024-06-18T03:12:44.159 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:12:44.160 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:12:44.160 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:12:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:12:43] "GET /metrics HTTP/1.1" 200 38211 "" "Prometheus/2.43.0" 2024-06-18T03:12:44.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:44 smithi163 ceph-mon[87697]: from='client.44508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:44.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:44 smithi163 ceph-mon[87697]: from='client.44514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:44.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:44 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:44.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:44 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/1187472688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:44.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:44 smithi163 ceph-mon[82879]: from='client.44508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:44.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:44 smithi163 ceph-mon[82879]: from='client.44514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:44.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:44 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:44.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:44 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/1187472688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:44 smithi186 ceph-mon[67751]: from='client.44508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:44 smithi186 ceph-mon[67751]: from='client.44514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:44 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:44 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/1187472688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:12:45.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:45 smithi163 ceph-mon[87697]: from='client.44520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:45.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:45 smithi163 ceph-mon[87697]: from='client.44532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:45.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:45 smithi163 ceph-mon[87697]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:45.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:45 smithi163 ceph-mon[82879]: from='client.44520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:45.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:45 smithi163 ceph-mon[82879]: from='client.44532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:45.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:45 smithi163 ceph-mon[82879]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:45.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:45 smithi186 ceph-mon[67751]: from='client.44520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:45.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:45 smithi186 ceph-mon[67751]: from='client.44532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:12:45.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:45 smithi186 ceph-mon[67751]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:47.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:47 smithi163 ceph-mon[87697]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:47.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:47 smithi163 ceph-mon[82879]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:47 smithi186 ceph-mon[67751]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:48.781 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:48 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-18T03:12:48.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:48 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-18T03:12:48.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:48 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-18T03:12:49.485 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:49 smithi186 ceph-mon[67751]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-18T03:12:49.485 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:49 smithi186 ceph-mon[67751]: Upgrade: osd.6 is safe to restart 2024-06-18T03:12:49.485 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:49 smithi186 ceph-mon[67751]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:49.485 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:49 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:49.485 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:49 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-18T03:12:49.485 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:49 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:12:49.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:49 smithi163 ceph-mon[82879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-18T03:12:49.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:49 smithi163 ceph-mon[82879]: Upgrade: osd.6 is safe to restart 2024-06-18T03:12:49.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:49 smithi163 ceph-mon[82879]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:49.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:49 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:49.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:49 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-18T03:12:49.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:49 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:12:49.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:49 smithi163 ceph-mon[87697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-18T03:12:49.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:49 smithi163 ceph-mon[87697]: Upgrade: osd.6 is safe to restart 2024-06-18T03:12:49.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:49 smithi163 ceph-mon[87697]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:49.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:49 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:49.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:49 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-18T03:12:49.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:49 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:12:50.630 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:50 smithi186 ceph-mon[67751]: Upgrade: Updating osd.6 2024-06-18T03:12:50.630 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:50 smithi186 ceph-mon[67751]: Deploying daemon osd.6 on smithi186 2024-06-18T03:12:50.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:50 smithi163 ceph-mon[82879]: Upgrade: Updating osd.6 2024-06-18T03:12:50.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:50 smithi163 ceph-mon[82879]: Deploying daemon osd.6 on smithi186 2024-06-18T03:12:50.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:50 smithi163 ceph-mon[87697]: Upgrade: Updating osd.6 2024-06-18T03:12:50.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:50 smithi163 ceph-mon[87697]: Deploying daemon osd.6 on smithi186 2024-06-18T03:12:51.413 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:51 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:51.413 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:51 smithi186 ceph-mon[67751]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:51.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:51 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:51.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:51 smithi163 ceph-mon[82879]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:51.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:51 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:12:51.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:51 smithi163 ceph-mon[87697]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:12:52.002 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:51 smithi186 systemd[1]: Stopping Ceph osd.6 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:12:52.318 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[35079]: 2024-06-18T03:12:52.001+0000 7f0df84f9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:12:52.318 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[35079]: 2024-06-18T03:12:52.001+0000 7f0df84f9700 -1 osd.6 142 *** Got signal Terminated *** 2024-06-18T03:12:52.318 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[35079]: 2024-06-18T03:12:52.001+0000 7f0df84f9700 -1 osd.6 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:12:52.724 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:52 smithi186 ceph-mon[67751]: osd.6 marked itself down and dead 2024-06-18T03:12:52.724 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:52 smithi186 ceph-mon[67751]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:52.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:52 smithi163 ceph-mon[82879]: osd.6 marked itself down and dead 2024-06-18T03:12:52.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:52 smithi163 ceph-mon[82879]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:52.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:52 smithi163 ceph-mon[87697]: osd.6 marked itself down and dead 2024-06-18T03:12:52.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:52 smithi163 ceph-mon[87697]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:12:53.005 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:52 smithi186 podman[88000]: 2024-06-18 03:12:52.724345958 +0000 UTC m=+0.824819767 container died 0fcc02594571f324181a8e8853fcf75d14b6c9ba64d3bdfcd549537f347499f0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, ceph=True, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vcs-type=git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_CLEAN=True, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, version=8) 2024-06-18T03:12:53.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:53 smithi186 podman[88000]: 2024-06-18 03:12:53.101601194 +0000 UTC m=+1.202075003 container cleanup 0fcc02594571f324181a8e8853fcf75d14b6c9ba64d3bdfcd549537f347499f0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6, ceph=True, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, name=centos-stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., release=754, io.openshift.expose-services=, com.redhat.component=centos-stream-container) 2024-06-18T03:12:53.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:53 smithi186 bash[88000]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6 2024-06-18T03:12:53.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:53 smithi186 ceph-mon[67751]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:12:53.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:53 smithi186 ceph-mon[67751]: osdmap e143: 8 total, 7 up, 8 in 2024-06-18T03:12:53.724 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:53 smithi186 podman[88028]: 2024-06-18 03:12:53.378353558 +0000 UTC m=+0.650448485 container remove 0fcc02594571f324181a8e8853fcf75d14b6c9ba64d3bdfcd549537f347499f0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_BRANCH=HEAD, vcs-type=git, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc.) 2024-06-18T03:12:53.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:53 smithi163 ceph-mon[87697]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:12:53.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:53 smithi163 ceph-mon[87697]: osdmap e143: 8 total, 7 up, 8 in 2024-06-18T03:12:53.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:53 smithi163 ceph-mon[82879]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:12:53.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:53 smithi163 ceph-mon[82879]: osdmap e143: 8 total, 7 up, 8 in 2024-06-18T03:12:53.990 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:53 smithi186 podman[88119]: 2024-06-18 03:12:53.626589769 +0000 UTC m=+0.017851482 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:12:53.990 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:53 smithi186 podman[88119]: 2024-06-18 03:12:53.754308392 +0000 UTC m=+0.145570108 container create a56c112e812b73a379276f1e40a2cbae7d348dc8a47ec0f6b6cacaeb776b2b00 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-06-18T03:12:54.260 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:53 smithi186 podman[88119]: 2024-06-18 03:12:53.990102143 +0000 UTC m=+0.381363865 container init a56c112e812b73a379276f1e40a2cbae7d348dc8a47ec0f6b6cacaeb776b2b00 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-deactivate, io.buildah.version=1.36.0, RELEASE=squid-3781e3a, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:12:54.260 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:53 smithi186 podman[88119]: 2024-06-18 03:12:53.993604243 +0000 UTC m=+0.384865959 container start a56c112e812b73a379276f1e40a2cbae7d348dc8a47ec0f6b6cacaeb776b2b00 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240612, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2) 2024-06-18T03:12:54.260 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:54 smithi186 podman[88119]: 2024-06-18 03:12:54.14012067 +0000 UTC m=+0.531382392 container attach a56c112e812b73a379276f1e40a2cbae7d348dc8a47ec0f6b6cacaeb776b2b00 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.build-date=20240612, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_CLEAN=True, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:12:54.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:12:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:12:53] "GET /metrics HTTP/1.1" 200 38211 "" "Prometheus/2.43.0" 2024-06-18T03:12:54.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:54 smithi186 ceph-mon[67751]: osdmap e144: 8 total, 7 up, 8 in 2024-06-18T03:12:54.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:54 smithi186 ceph-mon[67751]: pgmap v223: 161 pgs: 1 activating+undersized, 15 active+undersized, 1 peering, 4 stale+active+clean, 8 active+undersized+degraded, 132 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 41/723 objects degraded (5.671%) 2024-06-18T03:12:54.568 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:54 smithi186 podman[88119]: 2024-06-18 03:12:54.260398491 +0000 UTC m=+0.651660206 container died a56c112e812b73a379276f1e40a2cbae7d348dc8a47ec0f6b6cacaeb776b2b00 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-06-18T03:12:54.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:54 smithi163 ceph-mon[87697]: osdmap e144: 8 total, 7 up, 8 in 2024-06-18T03:12:54.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:54 smithi163 ceph-mon[87697]: pgmap v223: 161 pgs: 1 activating+undersized, 15 active+undersized, 1 peering, 4 stale+active+clean, 8 active+undersized+degraded, 132 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 41/723 objects degraded (5.671%) 2024-06-18T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:54 smithi163 ceph-mon[82879]: osdmap e144: 8 total, 7 up, 8 in 2024-06-18T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:54 smithi163 ceph-mon[82879]: pgmap v223: 161 pgs: 1 activating+undersized, 15 active+undersized, 1 peering, 4 stale+active+clean, 8 active+undersized+degraded, 132 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 41/723 objects degraded (5.671%) 2024-06-18T03:12:55.318 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:54 smithi186 podman[88141]: 2024-06-18 03:12:54.894179145 +0000 UTC m=+0.628110341 container remove a56c112e812b73a379276f1e40a2cbae7d348dc8a47ec0f6b6cacaeb776b2b00 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_CLEAN=True) 2024-06-18T03:12:55.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:54 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.6.service: Deactivated successfully. 2024-06-18T03:12:55.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:54 smithi186 systemd[1]: Stopped Ceph osd.6 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:12:55.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:54 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.6.service: Consumed 14.322s CPU time. 2024-06-18T03:12:55.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 systemd[1]: Starting Ceph osd.6 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:12:55.664 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:55 smithi186 ceph-mon[67751]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-06-18T03:12:55.664 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:55 smithi186 ceph-mon[67751]: Health check failed: Degraded data redundancy: 41/723 objects degraded (5.671%), 8 pgs degraded (PG_DEGRADED) 2024-06-18T03:12:55.664 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 podman[88241]: 2024-06-18 03:12:55.225725517 +0000 UTC m=+0.017110943 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:12:55.664 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 podman[88241]: 2024-06-18 03:12:55.336744501 +0000 UTC m=+0.128129927 container create 2ad78970464c52085950a7a63e225b5a8929110b2870709a8d731a6fdb849d89 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240612, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a, ceph=True, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-06-18T03:12:55.665 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 podman[88241]: 2024-06-18 03:12:55.564302953 +0000 UTC m=+0.355688379 container init 2ad78970464c52085950a7a63e225b5a8929110b2870709a8d731a6fdb849d89 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-activate, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux ) 2024-06-18T03:12:55.665 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 podman[88241]: 2024-06-18 03:12:55.567783208 +0000 UTC m=+0.359168637 container start 2ad78970464c52085950a7a63e225b5a8929110b2870709a8d731a6fdb849d89 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, GIT_CLEAN=True) 2024-06-18T03:12:55.665 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 podman[88241]: 2024-06-18 03:12:55.664248976 +0000 UTC m=+0.455634405 container attach 2ad78970464c52085950a7a63e225b5a8929110b2870709a8d731a6fdb849d89 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-activate, ceph=True, GIT_CLEAN=True, RELEASE=squid-3781e3a, org.label-schema.build-date=20240612, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-18T03:12:55.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:55 smithi163 ceph-mon[87697]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-06-18T03:12:55.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:55 smithi163 ceph-mon[87697]: Health check failed: Degraded data redundancy: 41/723 objects degraded (5.671%), 8 pgs degraded (PG_DEGRADED) 2024-06-18T03:12:55.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:55 smithi163 ceph-mon[82879]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-06-18T03:12:55.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:55 smithi163 ceph-mon[82879]: Health check failed: Degraded data redundancy: 41/723 objects degraded (5.671%), 8 pgs degraded (PG_DEGRADED) 2024-06-18T03:12:56.318 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-activate[88256]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-18T03:12:56.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 bash[88241]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-18T03:12:56.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-activate[88256]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-18T03:12:56.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 bash[88241]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-18T03:12:56.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-activate[88256]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-18T03:12:56.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 bash[88241]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-18T03:12:56.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-activate[88256]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-18T03:12:56.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 bash[88241]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-18T03:12:56.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-activate[88256]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-06-18T03:12:56.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 bash[88241]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-06-18T03:12:56.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-activate[88256]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-18T03:12:56.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 bash[88241]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-18T03:12:56.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-activate[88256]: --> ceph-volume raw activate successful for osd ID: 6 2024-06-18T03:12:56.320 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 bash[88241]: --> ceph-volume raw activate successful for osd ID: 6 2024-06-18T03:12:56.320 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:55 smithi186 podman[88241]: 2024-06-18 03:12:55.987697825 +0000 UTC m=+0.779083257 container died 2ad78970464c52085950a7a63e225b5a8929110b2870709a8d731a6fdb849d89 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-activate, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, GIT_BRANCH=HEAD) 2024-06-18T03:12:56.609 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:56 smithi186 ceph-mon[67751]: pgmap v224: 161 pgs: 1 activating+undersized, 32 active+undersized, 1 peering, 2 stale+active+clean, 16 active+undersized+degraded, 109 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 83/723 objects degraded (11.480%) 2024-06-18T03:12:56.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:56 smithi163 ceph-mon[87697]: pgmap v224: 161 pgs: 1 activating+undersized, 32 active+undersized, 1 peering, 2 stale+active+clean, 16 active+undersized+degraded, 109 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 83/723 objects degraded (11.480%) 2024-06-18T03:12:56.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:56 smithi163 ceph-mon[82879]: pgmap v224: 161 pgs: 1 activating+undersized, 32 active+undersized, 1 peering, 2 stale+active+clean, 16 active+undersized+degraded, 109 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 83/723 objects degraded (11.480%) 2024-06-18T03:12:56.933 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:56 smithi186 podman[88385]: 2024-06-18 03:12:56.610125073 +0000 UTC m=+0.616500340 container remove 2ad78970464c52085950a7a63e225b5a8929110b2870709a8d731a6fdb849d89 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-06-18T03:12:56.933 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:56 smithi186 podman[88445]: 2024-06-18 03:12:56.724315977 +0000 UTC m=+0.017252903 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:12:56.933 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:56 smithi186 podman[88445]: 2024-06-18 03:12:56.827605455 +0000 UTC m=+0.120542380 container create 1bfdef63cef0453b2d0006cfd96fcb5a4134365a98146b36c2bbe5abf6986403 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.build-date=20240612) 2024-06-18T03:12:57.318 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:57 smithi186 podman[88445]: 2024-06-18 03:12:57.054691367 +0000 UTC m=+0.347628293 container init 1bfdef63cef0453b2d0006cfd96fcb5a4134365a98146b36c2bbe5abf6986403 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a) 2024-06-18T03:12:57.318 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:57 smithi186 podman[88445]: 2024-06-18 03:12:57.057955618 +0000 UTC m=+0.350892545 container start 1bfdef63cef0453b2d0006cfd96fcb5a4134365a98146b36c2bbe5abf6986403 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-06-18T03:12:57.318 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:57 smithi186 ceph-osd[88464]: -- 172.21.15.186:0/412795941 <== mon.2 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55be5eaf7a40 con 0x55be5f8b6000 2024-06-18T03:12:57.318 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:57 smithi186 bash[88445]: 1bfdef63cef0453b2d0006cfd96fcb5a4134365a98146b36c2bbe5abf6986403 2024-06-18T03:12:57.319 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:57 smithi186 systemd[1]: Started Ceph osd.6 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:12:57.909 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:12:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:12:57.602+0000 7f99ab50b640 -1 Falling back to public interface 2024-06-18T03:12:58.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:58 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:58.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:58 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:58.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:58 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:58.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:58 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:58.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:58 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:58.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:58 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:59.598 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:59 smithi186 ceph-mon[67751]: pgmap v225: 161 pgs: 1 activating+undersized, 34 active+undersized, 1 peering, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-06-18T03:12:59.598 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:59.598 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:12:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:12:59.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:59 smithi163 ceph-mon[87697]: pgmap v225: 161 pgs: 1 activating+undersized, 34 active+undersized, 1 peering, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-06-18T03:12:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:59.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:12:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:12:59.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:59 smithi163 ceph-mon[82879]: pgmap v225: 161 pgs: 1 activating+undersized, 34 active+undersized, 1 peering, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-06-18T03:12:59.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:12:59.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:12:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:13:00.318 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:13:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:13:00.305+0000 7f99ab50b640 -1 osd.6 0 read_superblock omap replica is missing. 2024-06-18T03:13:00.764 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:00 smithi186 ceph-mon[67751]: from='osd.6 [v2:172.21.15.186:6816/3197380434,v1:172.21.15.186:6817/3197380434]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-18T03:13:00.764 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:00 smithi186 ceph-mon[67751]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-18T03:13:00.764 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:13:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:13:00.337+0000 7f99ab50b640 -1 osd.6 142 log_to_monitors true 2024-06-18T03:13:00.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:00 smithi163 ceph-mon[87697]: from='osd.6 [v2:172.21.15.186:6816/3197380434,v1:172.21.15.186:6817/3197380434]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-18T03:13:00.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:00 smithi163 ceph-mon[87697]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-18T03:13:00.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:00 smithi163 ceph-mon[82879]: from='osd.6 [v2:172.21.15.186:6816/3197380434,v1:172.21.15.186:6817/3197380434]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-18T03:13:00.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:00 smithi163 ceph-mon[82879]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-18T03:13:01.068 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:13:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:13:00.812+0000 7f999f2ae640 -1 osd.6 142 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T03:13:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[87697]: pgmap v226: 161 pgs: 35 active+undersized, 18 active+undersized+degraded, 108 active+clean; 585 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-06-18T03:13:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[87697]: Health check update: Degraded data redundancy: 85/723 objects degraded (11.757%), 18 pgs degraded (PG_DEGRADED) 2024-06-18T03:13:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[87697]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-06-18T03:13:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[87697]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-18T03:13:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[87697]: from='osd.6 [v2:172.21.15.186:6816/3197380434,v1:172.21.15.186:6817/3197380434]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:13:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[87697]: osdmap e145: 8 total, 7 up, 8 in 2024-06-18T03:13:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[87697]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:13:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[87697]: from='osd.6 ' entity='osd.6' 2024-06-18T03:13:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[82879]: pgmap v226: 161 pgs: 35 active+undersized, 18 active+undersized+degraded, 108 active+clean; 585 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-06-18T03:13:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[82879]: Health check update: Degraded data redundancy: 85/723 objects degraded (11.757%), 18 pgs degraded (PG_DEGRADED) 2024-06-18T03:13:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[82879]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-06-18T03:13:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[82879]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-18T03:13:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[82879]: from='osd.6 [v2:172.21.15.186:6816/3197380434,v1:172.21.15.186:6817/3197380434]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:13:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[82879]: osdmap e145: 8 total, 7 up, 8 in 2024-06-18T03:13:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[82879]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:13:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[82879]: from='osd.6 ' entity='osd.6' 2024-06-18T03:13:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:01.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:01 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:01 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:01 smithi186 ceph-mon[67751]: pgmap v226: 161 pgs: 35 active+undersized, 18 active+undersized+degraded, 108 active+clean; 585 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-06-18T03:13:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:01 smithi186 ceph-mon[67751]: Health check update: Degraded data redundancy: 85/723 objects degraded (11.757%), 18 pgs degraded (PG_DEGRADED) 2024-06-18T03:13:01.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:01 smithi186 ceph-mon[67751]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-06-18T03:13:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:01 smithi186 ceph-mon[67751]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-18T03:13:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:01 smithi186 ceph-mon[67751]: from='osd.6 [v2:172.21.15.186:6816/3197380434,v1:172.21.15.186:6817/3197380434]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:13:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:01 smithi186 ceph-mon[67751]: osdmap e145: 8 total, 7 up, 8 in 2024-06-18T03:13:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:01 smithi186 ceph-mon[67751]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:13:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:01 smithi186 ceph-mon[67751]: from='osd.6 ' entity='osd.6' 2024-06-18T03:13:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:01 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:01 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:02.518 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:02 smithi186 ceph-mon[67751]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:13:02.518 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:02 smithi186 ceph-mon[67751]: osd.6 [v2:172.21.15.186:6816/3197380434,v1:172.21.15.186:6817/3197380434] boot 2024-06-18T03:13:02.518 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:02 smithi186 ceph-mon[67751]: osdmap e146: 8 total, 8 up, 8 in 2024-06-18T03:13:02.518 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:02 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T03:13:02.518 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:02 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:02.519 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:02 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:02.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:02 smithi163 ceph-mon[87697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:13:02.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:02 smithi163 ceph-mon[87697]: osd.6 [v2:172.21.15.186:6816/3197380434,v1:172.21.15.186:6817/3197380434] boot 2024-06-18T03:13:02.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:02 smithi163 ceph-mon[87697]: osdmap e146: 8 total, 8 up, 8 in 2024-06-18T03:13:02.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:02 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T03:13:02.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:02 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:02.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:02 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:02 smithi163 ceph-mon[82879]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:13:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:02 smithi163 ceph-mon[82879]: osd.6 [v2:172.21.15.186:6816/3197380434,v1:172.21.15.186:6817/3197380434] boot 2024-06-18T03:13:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:02 smithi163 ceph-mon[82879]: osdmap e146: 8 total, 8 up, 8 in 2024-06-18T03:13:02.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:02 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-18T03:13:02.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:02 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:02.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:02 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:03.532 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:03 smithi186 ceph-mon[67751]: pgmap v229: 161 pgs: 1 peering, 34 active+undersized, 18 active+undersized+degraded, 108 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-06-18T03:13:03.532 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:03 smithi186 ceph-mon[67751]: osdmap e147: 8 total, 8 up, 8 in 2024-06-18T03:13:03.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:03 smithi163 ceph-mon[87697]: pgmap v229: 161 pgs: 1 peering, 34 active+undersized, 18 active+undersized+degraded, 108 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-06-18T03:13:03.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:03 smithi163 ceph-mon[87697]: osdmap e147: 8 total, 8 up, 8 in 2024-06-18T03:13:03.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:03 smithi163 ceph-mon[82879]: pgmap v229: 161 pgs: 1 peering, 34 active+undersized, 18 active+undersized+degraded, 108 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-06-18T03:13:03.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:03 smithi163 ceph-mon[82879]: osdmap e147: 8 total, 8 up, 8 in 2024-06-18T03:13:04.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:13:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:13:03] "GET /metrics HTTP/1.1" 200 38282 "" "Prometheus/2.43.0" 2024-06-18T03:13:05.543 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:13:05.225+0000 7f10a413f640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-06-18T03:13:05.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[82879]: pgmap v231: 161 pgs: 1 peering, 24 active+undersized, 12 active+undersized+degraded, 124 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/723 objects degraded (7.746%) 2024-06-18T03:13:05.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:05.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:05.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:05.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:13:05.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:05.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:13:05.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:05.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:05.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:05.544 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-18T03:13:05.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[87697]: pgmap v231: 161 pgs: 1 peering, 24 active+undersized, 12 active+undersized+degraded, 124 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/723 objects degraded (7.746%) 2024-06-18T03:13:05.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:05.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:05.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:05.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:13:05.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:05.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:13:05.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:05.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:05.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:05.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:05 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-18T03:13:05.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:05 smithi186 ceph-mon[67751]: pgmap v231: 161 pgs: 1 peering, 24 active+undersized, 12 active+undersized+degraded, 124 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/723 objects degraded (7.746%) 2024-06-18T03:13:05.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:05.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:05.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:05.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:13:05.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:05.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:13:05.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:05.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:05.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:05.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:05 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-18T03:13:06.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:06 smithi163 ceph-mon[82879]: pgmap v232: 161 pgs: 1 peering, 18 active+undersized, 9 active+undersized+degraded, 133 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 40/723 objects degraded (5.533%) 2024-06-18T03:13:06.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:06 smithi163 ceph-mon[82879]: pgmap v233: 161 pgs: 1 peering, 18 active+undersized, 9 active+undersized+degraded, 133 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 40/723 objects degraded (5.533%) 2024-06-18T03:13:06.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:06 smithi163 ceph-mon[82879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-18T03:13:06.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:06 smithi163 ceph-mon[82879]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-06-18T03:13:06.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:06 smithi163 ceph-mon[82879]: Health check update: Degraded data redundancy: 40/723 objects degraded (5.533%), 9 pgs degraded (PG_DEGRADED) 2024-06-18T03:13:06.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:06 smithi163 ceph-mon[87697]: pgmap v232: 161 pgs: 1 peering, 18 active+undersized, 9 active+undersized+degraded, 133 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 40/723 objects degraded (5.533%) 2024-06-18T03:13:06.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:06 smithi163 ceph-mon[87697]: pgmap v233: 161 pgs: 1 peering, 18 active+undersized, 9 active+undersized+degraded, 133 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 40/723 objects degraded (5.533%) 2024-06-18T03:13:06.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:06 smithi163 ceph-mon[87697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-18T03:13:06.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:06 smithi163 ceph-mon[87697]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-06-18T03:13:06.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:06 smithi163 ceph-mon[87697]: Health check update: Degraded data redundancy: 40/723 objects degraded (5.533%), 9 pgs degraded (PG_DEGRADED) 2024-06-18T03:13:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:06 smithi186 ceph-mon[67751]: pgmap v232: 161 pgs: 1 peering, 18 active+undersized, 9 active+undersized+degraded, 133 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 40/723 objects degraded (5.533%) 2024-06-18T03:13:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:06 smithi186 ceph-mon[67751]: pgmap v233: 161 pgs: 1 peering, 18 active+undersized, 9 active+undersized+degraded, 133 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 40/723 objects degraded (5.533%) 2024-06-18T03:13:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:06 smithi186 ceph-mon[67751]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-18T03:13:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:06 smithi186 ceph-mon[67751]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-06-18T03:13:06.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:06 smithi186 ceph-mon[67751]: Health check update: Degraded data redundancy: 40/723 objects degraded (5.533%), 9 pgs degraded (PG_DEGRADED) 2024-06-18T03:13:08.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:07 smithi163 ceph-mon[87697]: pgmap v234: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 774 B/s rd, 0 op/s 2024-06-18T03:13:08.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:07 smithi163 ceph-mon[82879]: pgmap v234: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 774 B/s rd, 0 op/s 2024-06-18T03:13:08.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:07 smithi186 ceph-mon[67751]: pgmap v234: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 774 B/s rd, 0 op/s 2024-06-18T03:13:09.220 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:08 smithi186 ceph-mon[67751]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/723 objects degraded (5.533%), 9 pgs degraded) 2024-06-18T03:13:09.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:08 smithi163 ceph-mon[87697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/723 objects degraded (5.533%), 9 pgs degraded) 2024-06-18T03:13:09.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:08 smithi163 ceph-mon[82879]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/723 objects degraded (5.533%), 9 pgs degraded) 2024-06-18T03:13:10.231 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:09 smithi186 ceph-mon[67751]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-06-18T03:13:10.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:09 smithi163 ceph-mon[87697]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-06-18T03:13:10.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:09 smithi163 ceph-mon[82879]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-06-18T03:13:11.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:10 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:11.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:10 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:10 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:12.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:11 smithi163 ceph-mon[87697]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:13:12.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:11 smithi163 ceph-mon[82879]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:13:12.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:11 smithi186 ceph-mon[67751]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:13:14.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:13:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:13:13] "GET /metrics HTTP/1.1" 200 38285 "" "Prometheus/2.43.0" 2024-06-18T03:13:14.518 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:13:14.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:14 smithi163 ceph-mon[87697]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:13:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:13:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:14 smithi163 ceph-mon[82879]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:13:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:13:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:14 smithi186 ceph-mon[67751]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:13:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:13:15.162 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:13:15.162 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (7m) 114s ago 29m 27.1M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:13:15.162 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (7m) 13s ago 29m 79.4M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:13:15.162 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 114s ago 29m 59.1M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:13:15.162 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (9m) 13s ago 34m 443M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:13:15.162 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (18m) 114s ago 36m 502M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:13:15.162 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (5m) 114s ago 36m 52.2M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:13:15.163 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (6m) 13s ago 34m 50.8M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:13:15.163 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (5m) 114s ago 34m 44.7M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:13:15.163 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (8m) 114s ago 30m 15.9M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:13:15.163 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (8m) 13s ago 30m 16.2M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:13:15.163 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (4m) 114s ago 34m 44.8M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:13:15.163 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (3m) 114s ago 33m 44.9M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:13:15.163 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (2m) 114s ago 33m 48.0M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 05782842a9a4 2024-06-18T03:13:15.163 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (118s) 114s ago 32m 11.8M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 526499193ea3 2024-06-18T03:13:15.163 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (84s) 13s ago 32m 54.1M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 2c0ea5619d38 2024-06-18T03:13:15.163 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (51s) 13s ago 32m 54.2M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 e1d8886de2c4 2024-06-18T03:13:15.163 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (18s) 13s ago 31m 11.1M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 1bfdef63cef0 2024-06-18T03:13:15.164 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (31m) 13s ago 31m 61.1M 3248M 17.2.0 e1d6a67b021e 0587ef1746d9 2024-06-18T03:13:15.164 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (8m) 13s ago 30m 71.7M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:13:15.164 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (29m) 114s ago 29m 81.0M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:13:15.164 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (29m) 13s ago 29m 82.3M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:13:15.552 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:13:15.553 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:13:15.553 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:13:15.553 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:13:15.553 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:13:15.553 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:13:15.553 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:13:15.553 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:13:15.553 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-06-18T03:13:15.553 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 7 2024-06-18T03:13:15.553 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:13:15.553 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:13:15.554 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:13:15.554 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:13:15.554 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:13:15.554 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-06-18T03:13:15.554 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 12 2024-06-18T03:13:15.554 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:13:15.554 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:13:15.881 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:13:15.881 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:13:15.881 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:13:15.881 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-06-18T03:13:15.881 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:13:15.881 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "7/8 daemons upgraded", 2024-06-18T03:13:15.881 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading osd daemons", 2024-06-18T03:13:15.881 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:13:15.881 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:13:16.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:16 smithi163 ceph-mon[87697]: from='client.44541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:16.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:16 smithi163 ceph-mon[87697]: from='client.54511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:16.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:16 smithi163 ceph-mon[87697]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-06-18T03:13:16.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:16 smithi163 ceph-mon[87697]: from='client.44553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:16.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:16 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/2260732255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:16.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:16 smithi163 ceph-mon[82879]: from='client.44541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:16.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:16 smithi163 ceph-mon[82879]: from='client.54511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:16.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:16 smithi163 ceph-mon[82879]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-06-18T03:13:16.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:16 smithi163 ceph-mon[82879]: from='client.44553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:16.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:16 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/2260732255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:16 smithi186 ceph-mon[67751]: from='client.44541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:16 smithi186 ceph-mon[67751]: from='client.54511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:16 smithi186 ceph-mon[67751]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-06-18T03:13:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:16 smithi186 ceph-mon[67751]: from='client.44553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:16.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:16 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/2260732255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:17.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:17 smithi163 ceph-mon[82879]: from='client.54529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:17.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:17 smithi163 ceph-mon[87697]: from='client.54529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:17 smithi186 ceph-mon[67751]: from='client.54529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:18.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:18 smithi163 ceph-mon[87697]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:18.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:18 smithi163 ceph-mon[82879]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:18.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:18 smithi186 ceph-mon[67751]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:19.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:19 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:19.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:19 smithi163 ceph-mon[87697]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:19 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:19 smithi163 ceph-mon[82879]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:19.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:19 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:19.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:19 smithi186 ceph-mon[67751]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:20.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:20 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:20.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:20 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-18T03:13:20.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:20 smithi186 ceph-mon[67751]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-18T03:13:20.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:20 smithi186 ceph-mon[67751]: Upgrade: osd.7 is safe to restart 2024-06-18T03:13:20.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:20 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:20.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:20 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-18T03:13:20.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:20 smithi163 ceph-mon[87697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-18T03:13:20.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:20 smithi163 ceph-mon[87697]: Upgrade: osd.7 is safe to restart 2024-06-18T03:13:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:20 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:20 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-18T03:13:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:20 smithi163 ceph-mon[82879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-18T03:13:20.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:20 smithi163 ceph-mon[82879]: Upgrade: osd.7 is safe to restart 2024-06-18T03:13:21.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:21 smithi163 ceph-mon[87697]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:21 smithi163 ceph-mon[82879]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:21.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:21 smithi186 ceph-mon[67751]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:22.927 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:22 smithi186 ceph-mon[67751]: Upgrade: Updating osd.7 2024-06-18T03:13:22.928 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:22 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:22.928 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:22 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-18T03:13:22.928 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:22 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:22.928 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:22 smithi186 ceph-mon[67751]: Deploying daemon osd.7 on smithi186 2024-06-18T03:13:23.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:22 smithi163 ceph-mon[87697]: Upgrade: Updating osd.7 2024-06-18T03:13:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:22 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:22 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-18T03:13:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:22 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:23.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:22 smithi163 ceph-mon[87697]: Deploying daemon osd.7 on smithi186 2024-06-18T03:13:23.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:22 smithi163 ceph-mon[82879]: Upgrade: Updating osd.7 2024-06-18T03:13:23.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:22 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:23.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:22 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-18T03:13:23.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:22 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:23.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:22 smithi163 ceph-mon[82879]: Deploying daemon osd.7 on smithi186 2024-06-18T03:13:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:23 smithi186 ceph-mon[67751]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:24.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:23 smithi163 ceph-mon[87697]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:24.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:23 smithi163 ceph-mon[82879]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:24.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:13:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:13:23] "GET /metrics HTTP/1.1" 200 38285 "" "Prometheus/2.43.0" 2024-06-18T03:13:24.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:24 smithi186 systemd[1]: Stopping Ceph osd.7 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:13:25.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:24 smithi163 ceph-mon[87697]: osd.7 marked itself down and dead 2024-06-18T03:13:25.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:24 smithi163 ceph-mon[82879]: osd.7 marked itself down and dead 2024-06-18T03:13:25.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:24 smithi186 ceph-mon[67751]: osd.7 marked itself down and dead 2024-06-18T03:13:25.318 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:24 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[37987]: 2024-06-18T03:13:24.890+0000 7f49b37ad700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:13:25.318 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:24 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[37987]: 2024-06-18T03:13:24.890+0000 7f49b37ad700 -1 osd.7 147 *** Got signal Terminated *** 2024-06-18T03:13:25.318 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:24 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[37987]: 2024-06-18T03:13:24.890+0000 7f49b37ad700 -1 osd.7 147 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:13:25.804 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:25 smithi186 podman[92501]: 2024-06-18 03:13:25.513379845 +0000 UTC m=+0.763091138 container died 0587ef1746d9dbf6fdb60ae89eb61e37c263a64cd7963daff7e040d241109367 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, vcs-type=git, version=8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., distribution-scope=public, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8) 2024-06-18T03:13:26.057 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:25 smithi186 ceph-mon[67751]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:13:26.057 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:25 smithi186 ceph-mon[67751]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-06-18T03:13:26.057 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:25 smithi186 ceph-mon[67751]: osdmap e148: 8 total, 7 up, 8 in 2024-06-18T03:13:26.057 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:25 smithi186 ceph-mon[67751]: pgmap v244: 161 pgs: 22 stale+active+clean, 139 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:13:26.057 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:25 smithi186 ceph-mon[67751]: osdmap e149: 8 total, 7 up, 8 in 2024-06-18T03:13:26.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:25 smithi163 ceph-mon[87697]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:13:26.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:25 smithi163 ceph-mon[87697]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-06-18T03:13:26.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:25 smithi163 ceph-mon[87697]: osdmap e148: 8 total, 7 up, 8 in 2024-06-18T03:13:26.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:25 smithi163 ceph-mon[87697]: pgmap v244: 161 pgs: 22 stale+active+clean, 139 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:13:26.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:25 smithi163 ceph-mon[87697]: osdmap e149: 8 total, 7 up, 8 in 2024-06-18T03:13:26.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:25 smithi163 ceph-mon[82879]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-18T03:13:26.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:25 smithi163 ceph-mon[82879]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-06-18T03:13:26.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:25 smithi163 ceph-mon[82879]: osdmap e148: 8 total, 7 up, 8 in 2024-06-18T03:13:26.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:25 smithi163 ceph-mon[82879]: pgmap v244: 161 pgs: 22 stale+active+clean, 139 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:13:26.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:25 smithi163 ceph-mon[82879]: osdmap e149: 8 total, 7 up, 8 in 2024-06-18T03:13:26.318 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:26 smithi186 podman[92501]: 2024-06-18 03:13:26.057622508 +0000 UTC m=+1.307333801 container cleanup 0587ef1746d9dbf6fdb60ae89eb61e37c263a64cd7963daff7e040d241109367 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, GIT_CLEAN=True, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8) 2024-06-18T03:13:26.318 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:26 smithi186 bash[92501]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7 2024-06-18T03:13:26.681 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:26 smithi186 podman[92527]: 2024-06-18 03:13:26.359137483 +0000 UTC m=+0.843601053 container remove 0587ef1746d9dbf6fdb60ae89eb61e37c263a64cd7963daff7e040d241109367 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7, vcs-type=git, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, release=754, version=8, RELEASE=HEAD, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , name=centos-stream, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0) 2024-06-18T03:13:26.947 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:26 smithi186 podman[92618]: 2024-06-18 03:13:26.585138303 +0000 UTC m=+0.017944636 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:13:26.947 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:26 smithi186 podman[92618]: 2024-06-18 03:13:26.710104687 +0000 UTC m=+0.142911012 container create d89d5b2ec0d0e9c5060f1ceb6d6948a9a4e09d6c1b7e84a9cef1453908e8f57e (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-06-18T03:13:26.947 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:26 smithi186 podman[92618]: 2024-06-18 03:13:26.94655735 +0000 UTC m=+0.379363693 container init d89d5b2ec0d0e9c5060f1ceb6d6948a9a4e09d6c1b7e84a9cef1453908e8f57e (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0) 2024-06-18T03:13:27.205 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:26 smithi186 podman[92618]: 2024-06-18 03:13:26.950318781 +0000 UTC m=+0.383125109 container start d89d5b2ec0d0e9c5060f1ceb6d6948a9a4e09d6c1b7e84a9cef1453908e8f57e (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD) 2024-06-18T03:13:27.205 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:27 smithi186 podman[92618]: 2024-06-18 03:13:27.029194563 +0000 UTC m=+0.462000889 container attach d89d5b2ec0d0e9c5060f1ceb6d6948a9a4e09d6c1b7e84a9cef1453908e8f57e (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-3781e3a, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:13:27.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:27 smithi163 ceph-mon[87697]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-18T03:13:27.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:27 smithi163 ceph-mon[87697]: Health check failed: Degraded data redundancy: 22/723 objects degraded (3.043%), 5 pgs degraded (PG_DEGRADED) 2024-06-18T03:13:27.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:27 smithi163 ceph-mon[82879]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-18T03:13:27.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:27 smithi163 ceph-mon[82879]: Health check failed: Degraded data redundancy: 22/723 objects degraded (3.043%), 5 pgs degraded (PG_DEGRADED) 2024-06-18T03:13:27.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:27 smithi186 ceph-mon[67751]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-18T03:13:27.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:27 smithi186 ceph-mon[67751]: Health check failed: Degraded data redundancy: 22/723 objects degraded (3.043%), 5 pgs degraded (PG_DEGRADED) 2024-06-18T03:13:27.568 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:27 smithi186 podman[92618]: 2024-06-18 03:13:27.205355409 +0000 UTC m=+0.638161743 container died d89d5b2ec0d0e9c5060f1ceb6d6948a9a4e09d6c1b7e84a9cef1453908e8f57e (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-06-18T03:13:27.990 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:27 smithi186 podman[92640]: 2024-06-18 03:13:27.77476605 +0000 UTC m=+0.563389176 container remove d89d5b2ec0d0e9c5060f1ceb6d6948a9a4e09d6c1b7e84a9cef1453908e8f57e (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3781e3a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240612) 2024-06-18T03:13:27.991 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:27 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.7.service: Deactivated successfully. 2024-06-18T03:13:27.991 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:27 smithi186 systemd[1]: Stopped Ceph osd.7 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:13:27.991 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:27 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.7.service: Consumed 16.427s CPU time. 2024-06-18T03:13:28.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:28 smithi186 ceph-mon[67751]: pgmap v246: 161 pgs: 8 active+undersized, 16 peering, 15 stale+active+clean, 5 active+undersized+degraded, 117 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 22/723 objects degraded (3.043%) 2024-06-18T03:13:28.318 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:27 smithi186 systemd[1]: Starting Ceph osd.7 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:13:28.318 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 podman[92736]: 2024-06-18 03:13:28.110571687 +0000 UTC m=+0.017498948 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:13:28.319 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 podman[92736]: 2024-06-18 03:13:28.208955032 +0000 UTC m=+0.115882279 container create b863d51d9d6e21a458ed74c7571da1665e134c201db9bfb8f1d87c6673bd17ad (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-3781e3a, org.label-schema.license=GPLv2) 2024-06-18T03:13:28.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:28 smithi163 ceph-mon[87697]: pgmap v246: 161 pgs: 8 active+undersized, 16 peering, 15 stale+active+clean, 5 active+undersized+degraded, 117 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 22/723 objects degraded (3.043%) 2024-06-18T03:13:28.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:28 smithi163 ceph-mon[82879]: pgmap v246: 161 pgs: 8 active+undersized, 16 peering, 15 stale+active+clean, 5 active+undersized+degraded, 117 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 22/723 objects degraded (3.043%) 2024-06-18T03:13:28.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 podman[92736]: 2024-06-18 03:13:28.469998378 +0000 UTC m=+0.376925639 container init b863d51d9d6e21a458ed74c7571da1665e134c201db9bfb8f1d87c6673bd17ad (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-activate, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-06-18T03:13:28.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 podman[92736]: 2024-06-18 03:13:28.473459289 +0000 UTC m=+0.380386544 container start b863d51d9d6e21a458ed74c7571da1665e134c201db9bfb8f1d87c6673bd17ad (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:13:28.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 podman[92736]: 2024-06-18 03:13:28.553228016 +0000 UTC m=+0.460155274 container attach b863d51d9d6e21a458ed74c7571da1665e134c201db9bfb8f1d87c6673bd17ad (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-06-18T03:13:29.206 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-activate[92751]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-18T03:13:29.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 bash[92736]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-18T03:13:29.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-activate[92751]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-18T03:13:29.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 bash[92736]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-18T03:13:29.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-activate[92751]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-18T03:13:29.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 bash[92736]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-18T03:13:29.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-activate[92751]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-18T03:13:29.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 bash[92736]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-18T03:13:29.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-activate[92751]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-06-18T03:13:29.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 bash[92736]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-06-18T03:13:29.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-activate[92751]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-18T03:13:29.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 bash[92736]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-18T03:13:29.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-activate[92751]: --> ceph-volume raw activate successful for osd ID: 7 2024-06-18T03:13:29.207 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 bash[92736]: --> ceph-volume raw activate successful for osd ID: 7 2024-06-18T03:13:29.208 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:28 smithi186 podman[92736]: 2024-06-18 03:13:28.89541484 +0000 UTC m=+0.802342087 container died b863d51d9d6e21a458ed74c7571da1665e134c201db9bfb8f1d87c6673bd17ad (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-activate, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True) 2024-06-18T03:13:29.482 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:29.482 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:13:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:29 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:29 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:13:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:13:29.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:29 smithi186 podman[92879]: 2024-06-18 03:13:29.482494343 +0000 UTC m=+0.576595358 container remove b863d51d9d6e21a458ed74c7571da1665e134c201db9bfb8f1d87c6673bd17ad (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-activate, RELEASE=squid-3781e3a, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-18T03:13:29.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:29 smithi186 podman[92937]: 2024-06-18 03:13:29.601740333 +0000 UTC m=+0.018038988 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:13:29.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:29 smithi186 podman[92937]: 2024-06-18 03:13:29.725145775 +0000 UTC m=+0.141444429 container create a7781eaeaec54baf2978aa94b21100fddbd74a3ebb13d40da15c8c472b849f3a (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-3781e3a) 2024-06-18T03:13:30.087 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:29 smithi186 podman[92937]: 2024-06-18 03:13:29.986181429 +0000 UTC m=+0.402480090 container init a7781eaeaec54baf2978aa94b21100fddbd74a3ebb13d40da15c8c472b849f3a (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3781e3a, ceph=True) 2024-06-18T03:13:30.087 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:29 smithi186 podman[92937]: 2024-06-18 03:13:29.989592897 +0000 UTC m=+0.405891552 container start a7781eaeaec54baf2978aa94b21100fddbd74a3ebb13d40da15c8c472b849f3a (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0) 2024-06-18T03:13:30.087 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:30 smithi186 ceph-osd[92956]: -- 172.21.15.186:0/2592156991 <== mon.2 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562b3ffada40 con 0x562b40d6c000 2024-06-18T03:13:30.350 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:30 smithi186 bash[92937]: a7781eaeaec54baf2978aa94b21100fddbd74a3ebb13d40da15c8c472b849f3a 2024-06-18T03:13:30.350 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:30 smithi186 systemd[1]: Started Ceph osd.7 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:13:30.741 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:30 smithi186 ceph-mon[67751]: pgmap v247: 161 pgs: 32 active+undersized, 16 peering, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-06-18T03:13:30.741 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:30 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:30.741 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:30 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:30.741 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:30 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:13:30.534+0000 7f1d3b332640 -1 Falling back to public interface 2024-06-18T03:13:30.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:30 smithi163 ceph-mon[87697]: pgmap v247: 161 pgs: 32 active+undersized, 16 peering, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-06-18T03:13:30.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:30 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:30.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:30 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:30 smithi163 ceph-mon[82879]: pgmap v247: 161 pgs: 32 active+undersized, 16 peering, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-06-18T03:13:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:30 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:30 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:31.709 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:31 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:31.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:31 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:31.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:31 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:32.663 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:32 smithi186 ceph-mon[67751]: pgmap v248: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 92/723 objects degraded (12.725%) 2024-06-18T03:13:32.663 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:32 smithi186 ceph-mon[67751]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-18T03:13:32.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:32 smithi163 ceph-mon[87697]: pgmap v248: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 92/723 objects degraded (12.725%) 2024-06-18T03:13:32.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:32 smithi163 ceph-mon[87697]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-18T03:13:32.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:32 smithi163 ceph-mon[82879]: pgmap v248: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 92/723 objects degraded (12.725%) 2024-06-18T03:13:32.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:32 smithi163 ceph-mon[82879]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-18T03:13:33.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:33 smithi186 ceph-mon[67751]: from='osd.7 [v2:172.21.15.186:6824/2079143682,v1:172.21.15.186:6825/2079143682]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-18T03:13:33.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:33 smithi186 ceph-mon[67751]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-18T03:13:33.568 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:13:33.284+0000 7f1d3b332640 -1 osd.7 0 read_superblock omap replica is missing. 2024-06-18T03:13:33.568 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:13:33.310+0000 7f1d3b332640 -1 osd.7 147 log_to_monitors true 2024-06-18T03:13:33.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:33 smithi163 ceph-mon[82879]: from='osd.7 [v2:172.21.15.186:6824/2079143682,v1:172.21.15.186:6825/2079143682]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-18T03:13:33.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:33 smithi163 ceph-mon[82879]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-18T03:13:33.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:33 smithi163 ceph-mon[87697]: from='osd.7 [v2:172.21.15.186:6824/2079143682,v1:172.21.15.186:6825/2079143682]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-18T03:13:33.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:33 smithi163 ceph-mon[87697]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-18T03:13:34.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:13:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:13:33] "GET /metrics HTTP/1.1" 200 38364 "" "Prometheus/2.43.0" 2024-06-18T03:13:34.318 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:13:33 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:13:33.837+0000 7f1d2f0d5640 -1 osd.7 147 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-18T03:13:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[82879]: pgmap v249: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 92/723 objects degraded (12.725%) 2024-06-18T03:13:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[82879]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-18T03:13:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[82879]: from='osd.7 [v2:172.21.15.186:6824/2079143682,v1:172.21.15.186:6825/2079143682]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:13:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[82879]: osdmap e150: 8 total, 7 up, 8 in 2024-06-18T03:13:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[82879]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:13:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[82879]: from='osd.7 ' entity='osd.7' 2024-06-18T03:13:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:34.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[87697]: pgmap v249: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 92/723 objects degraded (12.725%) 2024-06-18T03:13:34.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[87697]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-18T03:13:34.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[87697]: from='osd.7 [v2:172.21.15.186:6824/2079143682,v1:172.21.15.186:6825/2079143682]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:13:34.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[87697]: osdmap e150: 8 total, 7 up, 8 in 2024-06-18T03:13:34.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[87697]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:13:34.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[87697]: from='osd.7 ' entity='osd.7' 2024-06-18T03:13:34.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:34.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:34 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:34 smithi186 ceph-mon[67751]: pgmap v249: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 92/723 objects degraded (12.725%) 2024-06-18T03:13:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:34 smithi186 ceph-mon[67751]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-18T03:13:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:34 smithi186 ceph-mon[67751]: from='osd.7 [v2:172.21.15.186:6824/2079143682,v1:172.21.15.186:6825/2079143682]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:13:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:34 smithi186 ceph-mon[67751]: osdmap e150: 8 total, 7 up, 8 in 2024-06-18T03:13:34.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:34 smithi186 ceph-mon[67751]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi186", "root=default"]}]: dispatch 2024-06-18T03:13:34.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:34 smithi186 ceph-mon[67751]: from='osd.7 ' entity='osd.7' 2024-06-18T03:13:34.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:34.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:34 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:35.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:35 smithi186 ceph-mon[67751]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:13:35.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:35 smithi186 ceph-mon[67751]: osd.7 [v2:172.21.15.186:6824/2079143682,v1:172.21.15.186:6825/2079143682] boot 2024-06-18T03:13:35.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:35 smithi186 ceph-mon[67751]: osdmap e151: 8 total, 8 up, 8 in 2024-06-18T03:13:35.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:35 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T03:13:35.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:35 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:35.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:35 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:35.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:35 smithi163 ceph-mon[87697]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:13:35.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:35 smithi163 ceph-mon[87697]: osd.7 [v2:172.21.15.186:6824/2079143682,v1:172.21.15.186:6825/2079143682] boot 2024-06-18T03:13:35.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:35 smithi163 ceph-mon[87697]: osdmap e151: 8 total, 8 up, 8 in 2024-06-18T03:13:35.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:35 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T03:13:35.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:35 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:35.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:35 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:35.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:35 smithi163 ceph-mon[82879]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-18T03:13:35.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:35 smithi163 ceph-mon[82879]: osd.7 [v2:172.21.15.186:6824/2079143682,v1:172.21.15.186:6825/2079143682] boot 2024-06-18T03:13:35.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:35 smithi163 ceph-mon[82879]: osdmap e151: 8 total, 8 up, 8 in 2024-06-18T03:13:35.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:35 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-18T03:13:35.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:35 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:35.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:35 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:36.447 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:36 smithi186 ceph-mon[67751]: pgmap v252: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 92/723 objects degraded (12.725%) 2024-06-18T03:13:36.447 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:36 smithi186 ceph-mon[67751]: osdmap e152: 8 total, 8 up, 8 in 2024-06-18T03:13:36.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:36 smithi163 ceph-mon[82879]: pgmap v252: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 92/723 objects degraded (12.725%) 2024-06-18T03:13:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:36 smithi163 ceph-mon[82879]: osdmap e152: 8 total, 8 up, 8 in 2024-06-18T03:13:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:36 smithi163 ceph-mon[87697]: pgmap v252: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 92/723 objects degraded (12.725%) 2024-06-18T03:13:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:36 smithi163 ceph-mon[87697]: osdmap e152: 8 total, 8 up, 8 in 2024-06-18T03:13:37.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:37 smithi186 ceph-mon[67751]: Health check update: Degraded data redundancy: 92/723 objects degraded (12.725%), 24 pgs degraded (PG_DEGRADED) 2024-06-18T03:13:37.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:37 smithi163 ceph-mon[82879]: Health check update: Degraded data redundancy: 92/723 objects degraded (12.725%), 24 pgs degraded (PG_DEGRADED) 2024-06-18T03:13:37.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:37 smithi163 ceph-mon[87697]: Health check update: Degraded data redundancy: 92/723 objects degraded (12.725%), 24 pgs degraded (PG_DEGRADED) 2024-06-18T03:13:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: pgmap v254: 161 pgs: 10 peering, 29 active+undersized, 18 active+undersized+degraded, 104 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-06-18T03:13:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:13:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:13:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-06-18T03:13:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-06-18T03:13:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-06-18T03:13:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-06-18T03:13:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-06-18T03:13:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-06-18T03:13:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-06-18T03:13:38.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-06-18T03:13:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-06-18T03:13:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-06-18T03:13:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-06-18T03:13:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-06-18T03:13:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-06-18T03:13:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-06-18T03:13:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-06-18T03:13:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-06-18T03:13:38.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-06-18T03:13:38.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: pgmap v254: 161 pgs: 10 peering, 29 active+undersized, 18 active+undersized+degraded, 104 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-06-18T03:13:38.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:38.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:38.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:38.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:13:38.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:38.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:13:38.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:38.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:38.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:38.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:38.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:38.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-06-18T03:13:38.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-06-18T03:13:38.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-06-18T03:13:38.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-06-18T03:13:38.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-06-18T03:13:38.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-06-18T03:13:38.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-06-18T03:13:38.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-06-18T03:13:38.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-06-18T03:13:38.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-06-18T03:13:38.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-06-18T03:13:38.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-06-18T03:13:38.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-06-18T03:13:38.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-06-18T03:13:38.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-06-18T03:13:38.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-06-18T03:13:38.799 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:38 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-06-18T03:13:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: pgmap v254: 161 pgs: 10 peering, 29 active+undersized, 18 active+undersized+degraded, 104 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-06-18T03:13:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:38.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:13:38.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:38.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:13:38.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:38.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:38.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:38.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:38.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:38.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-06-18T03:13:38.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-06-18T03:13:38.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-06-18T03:13:38.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-06-18T03:13:38.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-06-18T03:13:38.822 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-06-18T03:13:38.822 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-06-18T03:13:38.822 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-06-18T03:13:38.822 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-06-18T03:13:38.822 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-06-18T03:13:38.822 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-06-18T03:13:38.823 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-06-18T03:13:38.823 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-06-18T03:13:38.823 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-06-18T03:13:38.823 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-06-18T03:13:38.823 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-06-18T03:13:38.824 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:38 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-06-18T03:13:39.721 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: pgmap v255: 161 pgs: 10 peering, 22 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 64/723 objects degraded (8.852%) 2024-06-18T03:13:39.722 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: pgmap v256: 161 pgs: 10 peering, 22 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 64/723 objects degraded (8.852%) 2024-06-18T03:13:39.722 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all osd 2024-06-18T03:13:39.722 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: Upgrade: Setting require_osd_release to 19 squid 2024-06-18T03:13:39.722 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-06-18T03:13:39.722 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-06-18T03:13:39.722 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: osdmap e153: 8 total, 8 up, 8 in 2024-06-18T03:13:39.722 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.722 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.722 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.722 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.723 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.724 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:13:39.724 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:13:39.724 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.724 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:13:39.724 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:13:39.724 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:13:39.724 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:13:39.725 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:13:39.726 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.726 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.726 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.726 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.726 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.726 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.726 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:13:39.726 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:13:39.726 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:39.726 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:13:39.726 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.726 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:39.726 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:13:39.727 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:39 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: pgmap v255: 161 pgs: 10 peering, 22 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 64/723 objects degraded (8.852%) 2024-06-18T03:13:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: pgmap v256: 161 pgs: 10 peering, 22 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 64/723 objects degraded (8.852%) 2024-06-18T03:13:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all osd 2024-06-18T03:13:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: Upgrade: Setting require_osd_release to 19 squid 2024-06-18T03:13:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-06-18T03:13:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-06-18T03:13:39.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: osdmap e153: 8 total, 8 up, 8 in 2024-06-18T03:13:39.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:13:39.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:13:39.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:13:39.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: pgmap v255: 161 pgs: 10 peering, 22 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 64/723 objects degraded (8.852%) 2024-06-18T03:13:39.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: pgmap v256: 161 pgs: 10 peering, 22 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 64/723 objects degraded (8.852%) 2024-06-18T03:13:39.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all osd 2024-06-18T03:13:39.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: Upgrade: Setting require_osd_release to 19 squid 2024-06-18T03:13:39.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-06-18T03:13:39.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-06-18T03:13:39.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: osdmap e153: 8 total, 8 up, 8 in 2024-06-18T03:13:39.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:13:39.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:13:39.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:13:39.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:13:39.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-18T03:13:39.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:13:39.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:13:39.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:13:39.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:13:39.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:13:39.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:13:39.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:13:39.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:13:39.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:13:39.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:13:39.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:13:39.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:13:39.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:13:39.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:13:39.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:13:39.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-18T03:13:39.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:13:39.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:13:39.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:13:39.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:13:39.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:13:39.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:13:39.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:13:39.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:13:39.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:13:39.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:13:39.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:13:39.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:13:39.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.801 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.801 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:13:39.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:13:39.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:39.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:13:39.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:39.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:13:39.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:13:39.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:13:39.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:13:39.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:39.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:13:39.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:39.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:13:39.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:13:39.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:39 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:40.737 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mds 2024-06-18T03:13:40.737 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:13:40.737 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:13:40.737 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all nfs 2024-06-18T03:13:40.737 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:40 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:13:40.737 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:40 smithi186 ceph-mon[67751]: Upgrade: Finalizing container_image settings 2024-06-18T03:13:40.737 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:40 smithi186 ceph-mon[67751]: Upgrade: Complete! 2024-06-18T03:13:40.737 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:40 smithi186 ceph-mon[67751]: pgmap v258: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:40.737 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:40 smithi186 ceph-mon[67751]: pgmap v259: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-06-18T03:13:40.737 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:40 smithi186 ceph-mon[67751]: pgmap v260: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:13:40.737 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:40 smithi186 ceph-mon[67751]: pgmap v261: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:13:40.737 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:40 smithi186 ceph-mon[67751]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 64/723 objects degraded (8.852%), 15 pgs degraded) 2024-06-18T03:13:40.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mds 2024-06-18T03:13:40.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:13:40.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:13:40.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all nfs 2024-06-18T03:13:40.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:13:40.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[82879]: Upgrade: Finalizing container_image settings 2024-06-18T03:13:40.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[82879]: Upgrade: Complete! 2024-06-18T03:13:40.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[82879]: pgmap v258: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:40.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[82879]: pgmap v259: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-06-18T03:13:40.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[82879]: pgmap v260: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:13:40.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[82879]: pgmap v261: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:13:40.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[82879]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 64/723 objects degraded (8.852%), 15 pgs degraded) 2024-06-18T03:13:40.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mds 2024-06-18T03:13:40.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:13:40.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:13:40.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all nfs 2024-06-18T03:13:40.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:13:40.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[87697]: Upgrade: Finalizing container_image settings 2024-06-18T03:13:40.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[87697]: Upgrade: Complete! 2024-06-18T03:13:40.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[87697]: pgmap v258: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:40.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[87697]: pgmap v259: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-06-18T03:13:40.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[87697]: pgmap v260: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:13:40.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[87697]: pgmap v261: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:13:40.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:40 smithi163 ceph-mon[87697]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 64/723 objects degraded (8.852%), 15 pgs degraded) 2024-06-18T03:13:41.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:41 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:41.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:41 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:41.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:41 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:42.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:42 smithi163 ceph-mon[87697]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-06-18T03:13:42.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:42 smithi163 ceph-mon[82879]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-06-18T03:13:42.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:42 smithi186 ceph-mon[67751]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-06-18T03:13:44.293 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:13:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:13:43] "GET /metrics HTTP/1.1" 200 38374 "" "Prometheus/2.43.0" 2024-06-18T03:13:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:44 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:44 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:13:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:44 smithi163 ceph-mon[82879]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:44 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:44.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:44 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:44.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:44 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:13:44.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:44 smithi163 ceph-mon[87697]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:44.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:44 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:44 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:44 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:13:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:44 smithi186 ceph-mon[67751]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:44 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:45.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:45 smithi163 ceph-mon[87697]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-06-18T03:13:45.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:45 smithi163 ceph-mon[82879]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-06-18T03:13:45.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:45 smithi186 ceph-mon[67751]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-06-18T03:13:46.517 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:46 smithi163 ceph-mon[82879]: from='client.44574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:46.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:46 smithi163 ceph-mon[87697]: from='client.44574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:46 smithi186 ceph-mon[67751]: from='client.44574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:46.885 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps' 2024-06-18T03:13:47.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:47 smithi163 ceph-mon[87697]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:47.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:47 smithi163 ceph-mon[82879]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:47 smithi186 ceph-mon[67751]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:49.021 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:13:49.021 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (8m) 2m ago 30m 27.1M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:13:49.021 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (8m) 14s ago 30m 80.0M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:13:49.021 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 2m ago 29m 59.1M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:13:49.021 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (9m) 14s ago 34m 443M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:13:49.021 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (19m) 2m ago 36m 502M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:13:49.021 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (6m) 2m ago 36m 52.2M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:13:49.021 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (7m) 14s ago 35m 51.8M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (6m) 2m ago 35m 44.7M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (9m) 2m ago 31m 15.9M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (9m) 14s ago 31m 15.9M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (4m) 2m ago 34m 44.8M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (4m) 2m ago 34m 44.9M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (3m) 2m ago 33m 48.0M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 05782842a9a4 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (2m) 2m ago 33m 11.8M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 526499193ea3 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (118s) 14s ago 33m 54.9M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 2c0ea5619d38 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (85s) 14s ago 32m 55.8M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 e1d8886de2c4 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (52s) 14s ago 32m 50.0M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 1bfdef63cef0 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (19s) 14s ago 31m 12.1M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 a7781eaeaec5 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (8m) 14s ago 31m 71.8M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (29m) 2m ago 29m 81.0M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:13:49.022 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (29m) 14s ago 29m 82.4M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:13:49.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:49 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:49.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:49 smithi163 ceph-mon[87697]: from='client.54544 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:49.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:49 smithi163 ceph-mon[87697]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:13:49.727 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:49 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:49.727 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:49 smithi163 ceph-mon[82879]: from='client.54544 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:49.727 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:49 smithi163 ceph-mon[82879]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:13:49.750 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-06-18T03:13:49.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:49 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:13:49.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:49 smithi186 ceph-mon[67751]: from='client.54544 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:49.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:49 smithi186 ceph-mon[67751]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:13:50.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:50 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:50.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:50 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:50.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:50 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:13:51.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:51 smithi163 ceph-mon[87697]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:51.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:51 smithi163 ceph-mon[82879]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:51.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:51 smithi186 ceph-mon[67751]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:51.907 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:13:52.558 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-06-18T03:13:52.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:52 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/774649562' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:52.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:52 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/774649562' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:52.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:52 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/774649562' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:53.784 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:53 smithi163 ceph-mon[87697]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:53.785 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:53 smithi163 ceph-mon[82879]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:53.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:53 smithi186 ceph-mon[67751]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:54.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:13:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:13:53] "GET /metrics HTTP/1.1" 200 38374 "" "Prometheus/2.43.0" 2024-06-18T03:13:54.629 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)" 2024-06-18T03:13:54.933 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:54 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/2792061948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:54.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:54 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/2792061948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:55.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:54 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/2792061948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:13:55.333 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade status' 2024-06-18T03:13:55.963 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:55 smithi163 ceph-mon[87697]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:55.964 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:55 smithi163 ceph-mon[82879]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:56.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:55 smithi186 ceph-mon[67751]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:57.355 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:13:57.356 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": null, 2024-06-18T03:13:57.356 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": false, 2024-06-18T03:13:57.356 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "", 2024-06-18T03:13:57.356 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:13:57.356 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": null, 2024-06-18T03:13:57.356 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "", 2024-06-18T03:13:57.356 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:13:57.356 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:13:57.680 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:57 smithi163 ceph-mon[87697]: from='client.54562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:57.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:57 smithi163 ceph-mon[87697]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:57.681 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:57 smithi163 ceph-mon[82879]: from='client.54562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:57.681 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:57 smithi163 ceph-mon[82879]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:57 smithi186 ceph-mon[67751]: from='client.54562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:13:57.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:57 smithi186 ceph-mon[67751]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:13:57.974 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph health detail' 2024-06-18T03:13:58.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:58 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:13:58.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:58 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:13:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:58 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:13:59.495 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:13:59 smithi163 ceph-mon[82879]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:59.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:13:59 smithi163 ceph-mon[87697]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:13:59.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:13:59 smithi186 ceph-mon[67751]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:14:00.003 INFO:teuthology.orchestra.run.smithi163.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:14:00.003 INFO:teuthology.orchestra.run.smithi163.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:14:00.003 INFO:teuthology.orchestra.run.smithi163.stdout: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in error state 2024-06-18T03:14:00.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:00 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/1388589849' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:14:00.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:00 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:00.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:00 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/1388589849' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:14:00.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:00 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:00.614 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-06-18T03:14:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:00 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/1388589849' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:14:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:00 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:01.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:01 smithi163 ceph-mon[82879]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 2 op/s 2024-06-18T03:14:01.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:01 smithi163 ceph-mon[87697]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 2 op/s 2024-06-18T03:14:01.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:01 smithi186 ceph-mon[67751]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 2 op/s 2024-06-18T03:14:03.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:03 smithi163 ceph-mon[87697]: from='client.44610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:03.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:03 smithi163 ceph-mon[87697]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T03:14:03.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:03 smithi163 ceph-mon[82879]: from='client.44610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:03.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:03 smithi163 ceph-mon[82879]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T03:14:03.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:03 smithi186 ceph-mon[67751]: from='client.44610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:03.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:03 smithi186 ceph-mon[67751]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-18T03:14:03.882 INFO:teuthology.orchestra.run.smithi163.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:04.257 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:14:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:14:03] "GET /metrics HTTP/1.1" 200 38373 "" "Prometheus/2.43.0" 2024-06-18T03:14:04.622 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-18T03:14:05.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[87697]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:05.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:05.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:05.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:05.137 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[87697]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 39 op/s 2024-06-18T03:14:05.137 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[87697]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 49 op/s 2024-06-18T03:14:05.137 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:05.137 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[87697]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:05.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[82879]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:05.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:05.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:05.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:05.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[82879]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 39 op/s 2024-06-18T03:14:05.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[82879]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 49 op/s 2024-06-18T03:14:05.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:05.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:04 smithi163 ceph-mon[82879]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:05.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:04 smithi186 ceph-mon[67751]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:05.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:05.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:05.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:05.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:04 smithi186 ceph-mon[67751]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 39 op/s 2024-06-18T03:14:05.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:04 smithi186 ceph-mon[67751]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 49 op/s 2024-06-18T03:14:05.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:04 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:05.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:04 smithi186 ceph-mon[67751]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:07.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 100 op/s 2024-06-18T03:14:07.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:14:07.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:14:07.228 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mgr 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mon 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all crash 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all osd 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.229 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mds 2024-06-18T03:14:07.230 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 100 op/s 2024-06-18T03:14:07.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:14:07.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:14:07.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:14:07.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mgr 2024-06-18T03:14:07.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mon 2024-06-18T03:14:07.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all crash 2024-06-18T03:14:07.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all osd 2024-06-18T03:14:07.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mds 2024-06-18T03:14:07.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:06 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 100 op/s 2024-06-18T03:14:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:14:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:14:07.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mgr 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mon 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all crash 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all osd 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:07.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mds 2024-06-18T03:14:07.320 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:06 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:07.681 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:14:08.343 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:14:08.344 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (8m) 2m ago 30m 27.1M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:14:08.344 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (8m) 34s ago 30m 80.0M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:14:08.344 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 2m ago 29m 59.1M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:14:08.344 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (9m) 34s ago 35m 443M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:14:08.344 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (19m) 2m ago 37m 502M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:14:08.344 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (6m) 2m ago 37m 52.2M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:14:08.344 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (7m) 34s ago 35m 51.8M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:14:08.345 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (6m) 2m ago 35m 44.7M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:14:08.345 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (9m) 2m ago 31m 15.9M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:14:08.345 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (9m) 34s ago 31m 15.9M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:14:08.345 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (5m) 2m ago 35m 44.8M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:14:08.345 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (4m) 2m ago 34m 44.9M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:14:08.345 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (3m) 2m ago 34m 48.0M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 05782842a9a4 2024-06-18T03:14:08.345 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (2m) 2m ago 33m 11.8M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 526499193ea3 2024-06-18T03:14:08.346 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (2m) 34s ago 33m 54.9M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 2c0ea5619d38 2024-06-18T03:14:08.346 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (104s) 34s ago 33m 55.8M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 e1d8886de2c4 2024-06-18T03:14:08.346 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (71s) 34s ago 32m 50.0M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 1bfdef63cef0 2024-06-18T03:14:08.346 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (38s) 34s ago 32m 12.1M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 a7781eaeaec5 2024-06-18T03:14:08.346 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (9m) 34s ago 31m 71.8M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:14:08.346 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (30m) 2m ago 30m 81.0M - 17.2.0 e1d6a67b021e e5e5a3425646 2024-06-18T03:14:08.346 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (30m) 34s ago 30m 82.4M - 17.2.0 e1d6a67b021e 5e910d80fed9 2024-06-18T03:14:08.764 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:14:08.765 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:14:08.765 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:14:08.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:14:08.765 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:14:08.765 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:14:08.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:14:08.765 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:14:08.765 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 8 2024-06-18T03:14:08.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:14:08.766 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:14:08.766 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-18T03:14:08.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:14:08.766 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:14:08.766 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-06-18T03:14:08.766 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 13 2024-06-18T03:14:08.766 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:14:08.766 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:14:09.111 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:14:09.112 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:14:09.112 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:14:09.112 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-06-18T03:14:09.112 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:14:09.112 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "0/2 daemons upgraded", 2024-06-18T03:14:09.112 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading rgw daemons", 2024-06-18T03:14:09.112 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:14:09.112 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:14:09.255 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[87697]: from='client.54580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:09.255 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[87697]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 0 B/s wr, 141 op/s 2024-06-18T03:14:09.255 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[87697]: from='client.44622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:09.256 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[87697]: from='client.54592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:09.256 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:09.256 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T03:14:09.256 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:09.256 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/3583171725' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:09.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[82879]: from='client.54580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:09.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[82879]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 0 B/s wr, 141 op/s 2024-06-18T03:14:09.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[82879]: from='client.44622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:09.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[82879]: from='client.54592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:09.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:09.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T03:14:09.257 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:09.257 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:08 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/3583171725' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:08 smithi186 ceph-mon[67751]: from='client.54580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:08 smithi186 ceph-mon[67751]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 0 B/s wr, 141 op/s 2024-06-18T03:14:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:08 smithi186 ceph-mon[67751]: from='client.44622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:08 smithi186 ceph-mon[67751]: from='client.54592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:09.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:08 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:08 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi163.mvqwty", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T03:14:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:08 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:09.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:08 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/3583171725' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:10.186 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:09 smithi163 ceph-mon[87697]: Upgrade: Updating rgw.foo.smithi163.mvqwty (1/2) 2024-06-18T03:14:10.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:09 smithi163 ceph-mon[87697]: Deploying daemon rgw.foo.smithi163.mvqwty on smithi163 2024-06-18T03:14:10.187 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:09 smithi163 ceph-mon[87697]: from='client.44640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:10.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:09 smithi163 ceph-mon[82879]: Upgrade: Updating rgw.foo.smithi163.mvqwty (1/2) 2024-06-18T03:14:10.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:09 smithi163 ceph-mon[82879]: Deploying daemon rgw.foo.smithi163.mvqwty on smithi163 2024-06-18T03:14:10.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:09 smithi163 ceph-mon[82879]: from='client.44640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:10.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:09 smithi186 ceph-mon[67751]: Upgrade: Updating rgw.foo.smithi163.mvqwty (1/2) 2024-06-18T03:14:10.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:09 smithi186 ceph-mon[67751]: Deploying daemon rgw.foo.smithi163.mvqwty on smithi163 2024-06-18T03:14:10.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:09 smithi186 ceph-mon[67751]: from='client.44640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:11.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:10 smithi163 ceph-mon[82879]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 0 B/s wr, 167 op/s 2024-06-18T03:14:11.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:10 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:11.141 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:10 smithi163 ceph-mon[87697]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 0 B/s wr, 167 op/s 2024-06-18T03:14:11.141 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:10 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:10 smithi186 ceph-mon[67751]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 0 B/s wr, 167 op/s 2024-06-18T03:14:11.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:10 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:13.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:13 smithi163 ceph-mon[87697]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 0 B/s wr, 153 op/s 2024-06-18T03:14:13.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:13.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:13.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:13 smithi163 ceph-mon[82879]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 0 B/s wr, 153 op/s 2024-06-18T03:14:13.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:13.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:13.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:13 smithi186 ceph-mon[67751]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 0 B/s wr, 153 op/s 2024-06-18T03:14:13.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:13.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:14.212 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:14:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:14:13] "GET /metrics HTTP/1.1" 200 38370 "" "Prometheus/2.43.0" 2024-06-18T03:14:14.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:14:14.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:14.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T03:14:14.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:14 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:14.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:14:14.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:14.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T03:14:14.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:14 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:14.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:14:14.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:14.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi186.vxkxtx", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-18T03:14:14.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:14 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:15.423 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:15 smithi186 ceph-mon[67751]: Upgrade: Updating rgw.foo.smithi186.vxkxtx (2/2) 2024-06-18T03:14:15.423 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:15 smithi186 ceph-mon[67751]: Deploying daemon rgw.foo.smithi186.vxkxtx on smithi186 2024-06-18T03:14:15.423 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:15 smithi186 ceph-mon[67751]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 0 B/s wr, 156 op/s 2024-06-18T03:14:15.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:15 smithi163 ceph-mon[87697]: Upgrade: Updating rgw.foo.smithi186.vxkxtx (2/2) 2024-06-18T03:14:15.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:15 smithi163 ceph-mon[87697]: Deploying daemon rgw.foo.smithi186.vxkxtx on smithi186 2024-06-18T03:14:15.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:15 smithi163 ceph-mon[87697]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 0 B/s wr, 156 op/s 2024-06-18T03:14:15.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:15 smithi163 ceph-mon[82879]: Upgrade: Updating rgw.foo.smithi186.vxkxtx (2/2) 2024-06-18T03:14:15.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:15 smithi163 ceph-mon[82879]: Deploying daemon rgw.foo.smithi186.vxkxtx on smithi186 2024-06-18T03:14:15.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:15 smithi163 ceph-mon[82879]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 0 B/s wr, 156 op/s 2024-06-18T03:14:17.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:17 smithi163 ceph-mon[87697]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 110 KiB/s rd, 85 B/s wr, 177 op/s 2024-06-18T03:14:17.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:17 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:17.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:17 smithi163 ceph-mon[82879]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 110 KiB/s rd, 85 B/s wr, 177 op/s 2024-06-18T03:14:17.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:17 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:17.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:17 smithi186 ceph-mon[67751]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 110 KiB/s rd, 85 B/s wr, 177 op/s 2024-06-18T03:14:17.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:17 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:18.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:18 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:18.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:18 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:18.567 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:18 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:19.514 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:19 smithi186 ceph-mon[67751]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 85 B/s wr, 179 op/s 2024-06-18T03:14:19.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:19 smithi163 ceph-mon[87697]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 85 B/s wr, 179 op/s 2024-06-18T03:14:19.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:19 smithi163 ceph-mon[82879]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 85 B/s wr, 179 op/s 2024-06-18T03:14:21.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:21 smithi163 ceph-mon[82879]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s rd, 85 B/s wr, 205 op/s 2024-06-18T03:14:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:21 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:21 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:21 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:21 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:21.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:21 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:21.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:21 smithi163 ceph-mon[87697]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s rd, 85 B/s wr, 205 op/s 2024-06-18T03:14:21.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:21 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:21.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:21 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:21.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:21 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:21.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:21 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:21.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:21 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:21.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:21 smithi186 ceph-mon[67751]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s rd, 85 B/s wr, 205 op/s 2024-06-18T03:14:21.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:21 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:21.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:21 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:21.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:21 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:21.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:21 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:21.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:21 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:22 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:22 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:22 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:22.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:22 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:22.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:22 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:22.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:22 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:22.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:22 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:22.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:22 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:22.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:22 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:22.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:22 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:22.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:22 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:22.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:22 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:23.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:23 smithi163 ceph-mon[87697]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 170 B/s wr, 237 op/s 2024-06-18T03:14:23.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:23 smithi163 ceph-mon[82879]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 170 B/s wr, 237 op/s 2024-06-18T03:14:23.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:23 smithi186 ceph-mon[67751]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 170 B/s wr, 237 op/s 2024-06-18T03:14:24.042 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:14:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:14:23] "GET /metrics HTTP/1.1" 200 38370 "" "Prometheus/2.43.0" 2024-06-18T03:14:25.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 247 op/s 2024-06-18T03:14:25.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:25.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:25.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:14:25.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi163.mvqwty"}]: dispatch 2024-06-18T03:14:25.570 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi163.mvqwty"}]': finished 2024-06-18T03:14:25.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi186.vxkxtx"}]: dispatch 2024-06-18T03:14:25.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi186.vxkxtx"}]': finished 2024-06-18T03:14:25.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.571 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:14:25.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:14:25.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:14:25.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:14:25.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:14:25.572 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:14:25.573 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-18T03:14:25.573 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:14:25.573 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:14:25.573 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:14:25.573 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-06-18T03:14:25.573 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:14:25.573 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:14:25.573 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.573 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:14:25.573 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:14:25.573 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:14:25.573 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:14:25.573 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:14:25.574 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:14:25.574 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:14:25.574 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.574 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.574 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.574 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.574 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.574 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.574 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:14:25.575 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:14:25.575 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:25.575 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:25.575 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.575 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:25.575 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:25.575 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:25 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 247 op/s 2024-06-18T03:14:25.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:25.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:25.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:14:25.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi163.mvqwty"}]: dispatch 2024-06-18T03:14:25.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi163.mvqwty"}]': finished 2024-06-18T03:14:25.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi186.vxkxtx"}]: dispatch 2024-06-18T03:14:25.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi186.vxkxtx"}]': finished 2024-06-18T03:14:25.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 247 op/s 2024-06-18T03:14:25.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi163.mvqwty"}]: dispatch 2024-06-18T03:14:25.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi163.mvqwty"}]': finished 2024-06-18T03:14:25.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi186.vxkxtx"}]: dispatch 2024-06-18T03:14:25.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi186.vxkxtx"}]': finished 2024-06-18T03:14:25.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:14:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:14:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:14:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:14:25.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:14:25.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:14:25.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-18T03:14:25.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:14:25.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:14:25.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:14:25.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-06-18T03:14:25.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:14:25.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:14:25.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:14:25.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:14:25.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:25.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-18T03:14:25.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-18T03:14:25.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-18T03:14:25.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-18T03:14:25.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-18T03:14:25.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-18T03:14:25.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-18T03:14:25.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-18T03:14:25.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-18T03:14:25.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-18T03:14:25.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-06-18T03:14:25.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-18T03:14:25.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-18T03:14:25.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-18T03:14:25.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-18T03:14:25.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:14:25.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:14:25.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:14:25.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:14:25.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:14:25.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:14:25.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:14:25.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:25.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:25.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:25.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:25.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-18T03:14:25.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:25.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:25.805 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:25.805 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:25.805 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:25 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:26.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 145 KiB/s rd, 192 B/s wr, 229 op/s 2024-06-18T03:14:26.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 137 KiB/s rd, 118 B/s wr, 216 op/s 2024-06-18T03:14:26.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all rgw 2024-06-18T03:14:26.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:14:26.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:14:26.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all nfs 2024-06-18T03:14:26.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:14:26.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: Upgrade: Finalizing container_image settings 2024-06-18T03:14:26.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: Upgrade: Complete! 2024-06-18T03:14:26.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 148 B/s wr, 204 op/s 2024-06-18T03:14:26.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 208 B/s wr, 151 op/s 2024-06-18T03:14:26.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: Checking dashboard <-> RGW credentials 2024-06-18T03:14:26.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 39 op/s 2024-06-18T03:14:26.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:26 smithi186 ceph-mon[67751]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:14:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 145 KiB/s rd, 192 B/s wr, 229 op/s 2024-06-18T03:14:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 137 KiB/s rd, 118 B/s wr, 216 op/s 2024-06-18T03:14:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all rgw 2024-06-18T03:14:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:14:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:14:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all nfs 2024-06-18T03:14:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:14:26.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: Upgrade: Finalizing container_image settings 2024-06-18T03:14:26.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: Upgrade: Complete! 2024-06-18T03:14:26.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 148 B/s wr, 204 op/s 2024-06-18T03:14:26.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 208 B/s wr, 151 op/s 2024-06-18T03:14:26.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: Checking dashboard <-> RGW credentials 2024-06-18T03:14:26.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 39 op/s 2024-06-18T03:14:26.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[82879]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:14:26.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 145 KiB/s rd, 192 B/s wr, 229 op/s 2024-06-18T03:14:26.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 137 KiB/s rd, 118 B/s wr, 216 op/s 2024-06-18T03:14:26.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all rgw 2024-06-18T03:14:26.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:14:26.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:14:26.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all nfs 2024-06-18T03:14:26.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all nvmeof 2024-06-18T03:14:26.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: Upgrade: Finalizing container_image settings 2024-06-18T03:14:26.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: Upgrade: Complete! 2024-06-18T03:14:26.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 148 B/s wr, 204 op/s 2024-06-18T03:14:26.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 208 B/s wr, 151 op/s 2024-06-18T03:14:26.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: Checking dashboard <-> RGW credentials 2024-06-18T03:14:26.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 39 op/s 2024-06-18T03:14:26.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:26 smithi163 ceph-mon[87697]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:14:28.595 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:28 smithi163 ceph-mon[87697]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 5.8 KiB/s rd, 6 op/s 2024-06-18T03:14:28.595 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:28 smithi163 ceph-mon[82879]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 5.8 KiB/s rd, 6 op/s 2024-06-18T03:14:28.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:28 smithi186 ceph-mon[67751]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 5.8 KiB/s rd, 6 op/s 2024-06-18T03:14:29.584 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:14:29.584 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:29 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:14:29.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:29 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:14:29.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:30.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:30 smithi163 ceph-mon[82879]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 6 op/s 2024-06-18T03:14:30.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:30 smithi163 ceph-mon[87697]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 6 op/s 2024-06-18T03:14:30.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:30 smithi186 ceph-mon[67751]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 6 op/s 2024-06-18T03:14:31.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:31 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:31.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:31 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:31.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:31 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:32.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:32 smithi163 ceph-mon[82879]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 5 op/s 2024-06-18T03:14:32.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:32 smithi163 ceph-mon[87697]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 5 op/s 2024-06-18T03:14:32.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:32 smithi186 ceph-mon[67751]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 5 op/s 2024-06-18T03:14:34.293 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:14:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:14:33] "GET /metrics HTTP/1.1" 200 38408 "" "Prometheus/2.43.0" 2024-06-18T03:14:34.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:34 smithi163 ceph-mon[87697]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 4 op/s 2024-06-18T03:14:34.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:34 smithi163 ceph-mon[82879]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 4 op/s 2024-06-18T03:14:34.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:34 smithi186 ceph-mon[67751]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 4 op/s 2024-06-18T03:14:36.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:36 smithi163 ceph-mon[87697]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-06-18T03:14:36.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:36 smithi163 ceph-mon[82879]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-06-18T03:14:36.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:36 smithi186 ceph-mon[67751]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-06-18T03:14:38.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:38 smithi163 ceph-mon[87697]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-06-18T03:14:38.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:38 smithi163 ceph-mon[82879]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-06-18T03:14:38.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:38 smithi186 ceph-mon[67751]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-06-18T03:14:40.163 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps' 2024-06-18T03:14:40.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:40 smithi163 ceph-mon[87697]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-06-18T03:14:40.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:40 smithi163 ceph-mon[82879]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-06-18T03:14:40.620 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:40 smithi186 ceph-mon[67751]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-06-18T03:14:41.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:41 smithi163 ceph-mon[87697]: from='client.54688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:41.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:41 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:41.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:41 smithi163 ceph-mon[82879]: from='client.54688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:41.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:41 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:41.632 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:41 smithi186 ceph-mon[67751]: from='client.54688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:41.633 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:41 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:42.133 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:14:42.133 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (9m) 21s ago 31m 27.1M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:14:42.134 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (9m) 21s ago 31m 79.1M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:14:42.134 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 21s ago 30m 63.4M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:14:42.134 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (10m) 21s ago 35m 444M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:14:42.134 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (20m) 21s ago 37m 507M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:14:42.134 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (7m) 21s ago 37m 61.6M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:14:42.134 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (8m) 21s ago 36m 53.1M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:14:42.134 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (6m) 21s ago 36m 49.2M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:14:42.134 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (10m) 21s ago 32m 15.5M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:14:42.134 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (10m) 21s ago 32m 15.9M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:14:42.134 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (5m) 21s ago 35m 54.4M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:14:42.134 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (5m) 21s ago 35m 55.3M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:14:42.134 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (4m) 21s ago 34m 56.1M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 05782842a9a4 2024-06-18T03:14:42.134 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (3m) 21s ago 34m 61.2M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 526499193ea3 2024-06-18T03:14:42.135 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (2m) 21s ago 34m 62.5M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 2c0ea5619d38 2024-06-18T03:14:42.135 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (2m) 21s ago 33m 60.8M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 e1d8886de2c4 2024-06-18T03:14:42.135 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (105s) 21s ago 33m 55.0M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 1bfdef63cef0 2024-06-18T03:14:42.135 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (72s) 21s ago 32m 57.5M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 a7781eaeaec5 2024-06-18T03:14:42.135 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (9m) 21s ago 32m 71.8M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:14:42.135 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (30s) 21s ago 30m 76.6M - 19.0.0-3124-g3781e3a5 af93c036e016 e5aaa8df5f66 2024-06-18T03:14:42.135 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (25s) 21s ago 30m 75.8M - 19.0.0-3124-g3781e3a5 af93c036e016 bc69579c3dbe 2024-06-18T03:14:42.431 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:42 smithi163 ceph-mon[87697]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:14:42.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:42 smithi163 ceph-mon[82879]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:14:42.747 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-06-18T03:14:42.766 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:42 smithi186 ceph-mon[67751]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:14:43.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:43 smithi163 ceph-mon[87697]: from='client.54694 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:43.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:14:43.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:43 smithi163 ceph-mon[82879]: from='client.54694 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:43.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:14:43.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:43 smithi186 ceph-mon[67751]: from='client.54694 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:43.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:14:44.108 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:14:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:14:43] "GET /metrics HTTP/1.1" 200 38408 "" "Prometheus/2.43.0" 2024-06-18T03:14:44.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:44 smithi163 ceph-mon[87697]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:14:44.361 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:44 smithi163 ceph-mon[82879]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:14:44.757 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:14:44.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:44 smithi186 ceph-mon[67751]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:14:45.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-06-18T03:14:45.630 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:45 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/4282511117' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:45.630 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:45 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/4282511117' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:45.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:45 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/4282511117' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:46.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:46 smithi163 ceph-mon[82879]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:14:46.609 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:46 smithi163 ceph-mon[87697]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:14:46.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:46 smithi186 ceph-mon[67751]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:14:47.356 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)" 2024-06-18T03:14:47.640 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:47 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/64383811' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:47.641 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:47 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/64383811' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:47.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:47 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/64383811' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:47.947 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade status' 2024-06-18T03:14:48.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:48 smithi163 ceph-mon[87697]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:14:48.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:48 smithi163 ceph-mon[82879]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:14:48.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:48 smithi186 ceph-mon[67751]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:14:49.872 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:14:49.873 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": null, 2024-06-18T03:14:49.873 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": false, 2024-06-18T03:14:49.873 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "", 2024-06-18T03:14:49.873 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [], 2024-06-18T03:14:49.873 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": null, 2024-06-18T03:14:49.873 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "", 2024-06-18T03:14:49.873 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:14:49.873 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:14:50.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:50 smithi163 ceph-mon[82879]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:14:50.446 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:50 smithi163 ceph-mon[87697]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:14:50.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph health detail' 2024-06-18T03:14:50.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:50 smithi186 ceph-mon[67751]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:14:51.769 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:51 smithi163 ceph-mon[82879]: from='client.44748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:51.769 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:51 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:51.769 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:51 smithi163 ceph-mon[87697]: from='client.44748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:51.770 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:51 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:51.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:51 smithi186 ceph-mon[67751]: from='client.44748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:51.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:51 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:14:52.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:52 smithi186 ceph-mon[67751]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:14:52.608 INFO:teuthology.orchestra.run.smithi163.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:14:52.608 INFO:teuthology.orchestra.run.smithi163.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:14:52.608 INFO:teuthology.orchestra.run.smithi163.stdout: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in error state 2024-06-18T03:14:52.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:52 smithi163 ceph-mon[87697]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:14:52.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:52 smithi163 ceph-mon[82879]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:14:53.328 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-06-18T03:14:53.638 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:53 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/4023021584' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:14:53.639 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:53 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/4023021584' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:14:53.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:53 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/4023021584' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:14:53.912 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:14:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:14:53] "GET /metrics HTTP/1.1" 200 38408 "" "Prometheus/2.43.0" 2024-06-18T03:14:54.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:54 smithi163 ceph-mon[87697]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:14:54.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:54 smithi163 ceph-mon[82879]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:14:54.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:54 smithi186 ceph-mon[67751]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:14:55.377 INFO:teuthology.orchestra.run.smithi163.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:56.076 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-18T03:14:56.086 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi163.front.sepia.ceph.com 2024-06-18T03:14:56.087 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-06-18T03:14:56.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[87697]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:14:56.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[87697]: from='client.54724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:56.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[87697]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:56.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:56.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:56.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:56.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[87697]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 0 op/s 2024-06-18T03:14:56.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[87697]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-06-18T03:14:56.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:56.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[82879]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:14:56.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[82879]: from='client.54724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:56.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[82879]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:56.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:56.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:56.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:56.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[82879]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 0 op/s 2024-06-18T03:14:56.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[82879]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-06-18T03:14:56.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:56 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:56 smithi186 ceph-mon[67751]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:14:56.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:56 smithi186 ceph-mon[67751]: from='client.54724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:14:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:56 smithi186 ceph-mon[67751]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:56 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:56 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:14:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:56 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:14:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:56 smithi186 ceph-mon[67751]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 0 op/s 2024-06-18T03:14:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:56 smithi186 ceph-mon[67751]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-06-18T03:14:56.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:56 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:57.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:14:57.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.469 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:57.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:14:57.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.470 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:57 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:14:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-18T03:14:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:57.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:14:57.820 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:57 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:14:58.632 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[87697]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:14:58.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[87697]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:14:58.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mgr 2024-06-18T03:14:58.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mon 2024-06-18T03:14:58.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all crash 2024-06-18T03:14:58.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all osd 2024-06-18T03:14:58.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all mds 2024-06-18T03:14:58.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all rgw 2024-06-18T03:14:58.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:14:58.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-18T03:14:58.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[87697]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:14:58.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[87697]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:14:58.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:14:58.634 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[82879]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:14:58.634 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[82879]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:14:58.634 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mgr 2024-06-18T03:14:58.634 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mon 2024-06-18T03:14:58.634 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all crash 2024-06-18T03:14:58.634 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all osd 2024-06-18T03:14:58.634 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all mds 2024-06-18T03:14:58.634 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all rgw 2024-06-18T03:14:58.634 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:14:58.634 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-18T03:14:58.635 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[82879]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:14:58.635 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[82879]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:14:58.635 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:14:58 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:14:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:58 smithi186 ceph-mon[67751]: Upgrade: Target is version 19.0.0-3124-g3781e3a5 (squid) 2024-06-18T03:14:58.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:58 smithi186 ceph-mon[67751]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:64b08d1c422bf6aa753839cabed8a69924a54ceb6ef0660c2e561d19ca874314'] 2024-06-18T03:14:58.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:58 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mgr 2024-06-18T03:14:58.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:58 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mon 2024-06-18T03:14:58.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:58 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all crash 2024-06-18T03:14:58.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:58 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all osd 2024-06-18T03:14:58.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:58 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all mds 2024-06-18T03:14:58.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:58 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all rgw 2024-06-18T03:14:58.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:58 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all rbd-mirror 2024-06-18T03:14:58.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:58 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-18T03:14:58.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:58 smithi186 ceph-mon[67751]: Upgrade: Setting container_image for all ceph-exporter 2024-06-18T03:14:58.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:58 smithi186 ceph-mon[67751]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:14:58.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:14:58 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:15:00.186 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:15:00.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:00 smithi163 ceph-mon[87697]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-06-18T03:15:00.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:00 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:15:00.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:00 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:15:00.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:00 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:15:00.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:00 smithi163 ceph-mon[82879]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-06-18T03:15:00.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:00 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:15:00.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:00 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:15:00.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:00 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:15:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:00 smithi186 ceph-mon[67751]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-06-18T03:15:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:00 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:15:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:00 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi163.ucaxbw", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-18T03:15:00.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:00 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:15:00.850 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:15:00.851 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (9m) 40s ago 31m 27.1M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:15:00.851 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (9m) 40s ago 31m 79.1M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:15:00.851 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 40s ago 30m 63.4M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:15:00.851 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (10m) 40s ago 36m 444M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:15:00.851 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (20m) 40s ago 37m 507M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:15:00.851 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (7m) 40s ago 38m 61.6M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:15:00.851 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (8m) 40s ago 36m 53.1M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:15:00.851 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (7m) 40s ago 36m 49.2M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:15:00.851 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (10m) 40s ago 32m 15.5M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:15:00.851 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (10m) 40s ago 32m 15.9M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:15:00.851 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (6m) 40s ago 35m 54.4M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:15:00.851 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (5m) 40s ago 35m 55.3M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:15:00.851 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (4m) 40s ago 35m 56.1M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 05782842a9a4 2024-06-18T03:15:00.852 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (3m) 40s ago 34m 61.2M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 526499193ea3 2024-06-18T03:15:00.852 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (3m) 40s ago 34m 62.5M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 2c0ea5619d38 2024-06-18T03:15:00.852 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (2m) 40s ago 33m 60.8M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 e1d8886de2c4 2024-06-18T03:15:00.852 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (2m) 40s ago 33m 55.0M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 1bfdef63cef0 2024-06-18T03:15:00.852 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (91s) 40s ago 33m 57.5M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 a7781eaeaec5 2024-06-18T03:15:00.852 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (10m) 40s ago 32m 71.8M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:15:00.852 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (49s) 40s ago 31m 76.6M - 19.0.0-3124-g3781e3a5 af93c036e016 e5aaa8df5f66 2024-06-18T03:15:00.852 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (44s) 40s ago 30m 75.8M - 19.0.0-3124-g3781e3a5 af93c036e016 bc69579c3dbe 2024-06-18T03:15:01.248 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:15:01.248 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:15:01.248 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:15:01.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:15:01.248 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:15:01.248 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:15:01.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:15:01.248 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:15:01.248 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 8 2024-06-18T03:15:01.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:15:01.248 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:15:01.248 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:15:01.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:15:01.249 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:15:01.249 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 15 2024-06-18T03:15:01.249 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:15:01.249 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:15:01.609 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:15:01.609 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:15:01.609 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:15:01.609 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-18T03:15:01.609 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [ 2024-06-18T03:15:01.609 INFO:teuthology.orchestra.run.smithi163.stdout: "osd", 2024-06-18T03:15:01.609 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr", 2024-06-18T03:15:01.609 INFO:teuthology.orchestra.run.smithi163.stdout: "mon", 2024-06-18T03:15:01.609 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw" 2024-06-18T03:15:01.610 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2024-06-18T03:15:01.610 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "15/21 daemons upgraded", 2024-06-18T03:15:01.610 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Currently upgrading iscsi daemons", 2024-06-18T03:15:01.610 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": false 2024-06-18T03:15:01.610 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:15:01.679 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:01 smithi163 ceph-mon[87697]: Upgrade: Updating iscsi.foo.smithi163.ucaxbw 2024-06-18T03:15:01.679 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:01 smithi163 ceph-mon[87697]: Deploying daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:15:01.679 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:01 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:01.680 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:01 smithi163 ceph-mon[87697]: from='client.54730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:01.680 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:01 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/3914562845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:15:01.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:01 smithi163 ceph-mon[82879]: Upgrade: Updating iscsi.foo.smithi163.ucaxbw 2024-06-18T03:15:01.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:01 smithi163 ceph-mon[82879]: Deploying daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:15:01.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:01 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:01.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:01 smithi163 ceph-mon[82879]: from='client.54730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:01.680 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:01 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/3914562845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:15:01.821 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:01 smithi186 ceph-mon[67751]: Upgrade: Updating iscsi.foo.smithi163.ucaxbw 2024-06-18T03:15:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:01 smithi186 ceph-mon[67751]: Deploying daemon iscsi.foo.smithi163.ucaxbw on smithi163 2024-06-18T03:15:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:01 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:01 smithi186 ceph-mon[67751]: from='client.54730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:01.822 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:01 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/3914562845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:15:02.024 INFO:teuthology.orchestra.run.smithi163.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-06-18T03:15:02.024 INFO:teuthology.orchestra.run.smithi163.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:15:02.024 INFO:teuthology.orchestra.run.smithi163.stdout: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in error state 2024-06-18T03:15:02.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:02 smithi163 ceph-mon[87697]: from='client.44772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:02.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:02 smithi163 ceph-mon[87697]: from='client.54742 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:02.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:02 smithi163 ceph-mon[87697]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:02.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:02 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/2215484738' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:15:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:02 smithi163 ceph-mon[82879]: from='client.44772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:02 smithi163 ceph-mon[82879]: from='client.54742 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:02 smithi163 ceph-mon[82879]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:02.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:02 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/2215484738' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:15:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:02 smithi186 ceph-mon[67751]: from='client.44772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:02 smithi186 ceph-mon[67751]: from='client.54742 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:02 smithi186 ceph-mon[67751]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:02.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:02 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/2215484738' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:15:03.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:03 smithi163 ceph-mon[87697]: from='client.44790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:03.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:03 smithi163 ceph-mon[82879]: from='client.44790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:03.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:03 smithi186 ceph-mon[67751]: from='client.44790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:04.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:15:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:15:03] "GET /metrics HTTP/1.1" 200 38409 "" "Prometheus/2.43.0" 2024-06-18T03:15:04.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:04 smithi163 ceph-mon[87697]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:04.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:04 smithi163 ceph-mon[82879]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:04.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:04 smithi186 ceph-mon[67751]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:05.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:05 smithi163 ceph-mon[87697]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:15:05.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:05 smithi163 ceph-mon[82879]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:15:05.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:05 smithi186 ceph-mon[67751]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:15:07.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:07 smithi163 ceph-mon[87697]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:07.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:07 smithi163 ceph-mon[82879]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:07.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:07 smithi186 ceph-mon[67751]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:09.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:09 smithi163 ceph-mon[82879]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:09.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:09 smithi163 ceph-mon[87697]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:09.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:09 smithi186 ceph-mon[67751]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:10.720 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:10 smithi186 ceph-mon[67751]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:10.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:10 smithi163 ceph-mon[87697]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:10.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:10 smithi163 ceph-mon[82879]: from='client.15360 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:11.735 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:11 smithi186 ceph-mon[67751]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:11.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:11 smithi163 ceph-mon[87697]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:11.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:11 smithi163 ceph-mon[82879]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:12.542 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:15:12 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: 2024-06-18T03:15:12.498+0000 7f10960e3640 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi163.ucaxbw on host smithi163 failed. 2024-06-18T03:15:13.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[87697]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi163.ucaxbw on host smithi163 failed. 2024-06-18T03:15:13.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:15:13.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[87697]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 920 B/s rd, 0 op/s 2024-06-18T03:15:13.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:15:13.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:15:13.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[87697]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:15:13.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[87697]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:15:13.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:15:13.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:15:13.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[82879]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi163.ucaxbw on host smithi163 failed. 2024-06-18T03:15:13.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:15:13.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[82879]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 920 B/s rd, 0 op/s 2024-06-18T03:15:13.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:15:13.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:15:13.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[82879]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:15:13.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[82879]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:15:13.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:15:13.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:15:13.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:13 smithi186 ceph-mon[67751]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi163.ucaxbw on host smithi163 failed. 2024-06-18T03:15:13.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:15:13.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:13 smithi186 ceph-mon[67751]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 920 B/s rd, 0 op/s 2024-06-18T03:15:13.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:15:13.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:15:13.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:13 smithi186 ceph-mon[67751]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-18T03:15:13.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:13 smithi186 ceph-mon[67751]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-18T03:15:13.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:15:13.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:15:14.293 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:15:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:15:13] "GET /metrics HTTP/1.1" 200 38406 "" "Prometheus/2.43.0" 2024-06-18T03:15:14.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:14 smithi163 ceph-mon[87697]: Health check failed: Upgrading daemon iscsi.foo.smithi163.ucaxbw on host smithi163 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-06-18T03:15:14.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:14 smithi163 ceph-mon[82879]: Health check failed: Upgrading daemon iscsi.foo.smithi163.ucaxbw on host smithi163 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-06-18T03:15:14.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:14 smithi186 ceph-mon[67751]: Health check failed: Upgrading daemon iscsi.foo.smithi163.ucaxbw on host smithi163 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-06-18T03:15:15.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:15 smithi163 ceph-mon[87697]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 718 B/s rd, 0 op/s 2024-06-18T03:15:15.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:15 smithi163 ceph-mon[82879]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 718 B/s rd, 0 op/s 2024-06-18T03:15:15.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:15 smithi186 ceph-mon[67751]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 718 B/s rd, 0 op/s 2024-06-18T03:15:17.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:17 smithi163 ceph-mon[87697]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-06-18T03:15:17.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:17 smithi163 ceph-mon[82879]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-06-18T03:15:17.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:17 smithi186 ceph-mon[67751]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-06-18T03:15:19.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:19 smithi163 ceph-mon[87697]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 143 B/s rd, 0 op/s 2024-06-18T03:15:19.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:19 smithi163 ceph-mon[82879]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 143 B/s rd, 0 op/s 2024-06-18T03:15:19.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:19 smithi186 ceph-mon[67751]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 143 B/s rd, 0 op/s 2024-06-18T03:15:21.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:21 smithi163 ceph-mon[87697]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-06-18T03:15:21.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:21 smithi163 ceph-mon[82879]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-06-18T03:15:21.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:21 smithi186 ceph-mon[67751]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-06-18T03:15:23.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:23 smithi163 ceph-mon[87697]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-06-18T03:15:23.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:23 smithi163 ceph-mon[82879]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-06-18T03:15:23.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:23 smithi186 ceph-mon[67751]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-06-18T03:15:24.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:15:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:15:23] "GET /metrics HTTP/1.1" 200 38406 "" "Prometheus/2.43.0" 2024-06-18T03:15:24.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:24 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/3581494937' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T03:15:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:24 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/4211167256' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3495931855"}]: dispatch 2024-06-18T03:15:24.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:24 smithi163 ceph-mon[87697]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3495931855"}]: dispatch 2024-06-18T03:15:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:24 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/3581494937' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T03:15:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:24 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/4211167256' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3495931855"}]: dispatch 2024-06-18T03:15:24.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:24 smithi163 ceph-mon[82879]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3495931855"}]: dispatch 2024-06-18T03:15:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:24 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/3581494937' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-18T03:15:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:24 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/4211167256' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3495931855"}]: dispatch 2024-06-18T03:15:24.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:24 smithi186 ceph-mon[67751]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3495931855"}]: dispatch 2024-06-18T03:15:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:25 smithi186 ceph-mon[67751]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-06-18T03:15:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:25 smithi186 ceph-mon[67751]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3495931855"}]': finished 2024-06-18T03:15:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:25 smithi186 ceph-mon[67751]: osdmap e154: 8 total, 8 up, 8 in 2024-06-18T03:15:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:25 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/3326824023' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2268975469"}]: dispatch 2024-06-18T03:15:25.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:25 smithi186 ceph-mon[67751]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2268975469"}]: dispatch 2024-06-18T03:15:26.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:25 smithi163 ceph-mon[87697]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-06-18T03:15:26.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:25 smithi163 ceph-mon[87697]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3495931855"}]': finished 2024-06-18T03:15:26.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:25 smithi163 ceph-mon[87697]: osdmap e154: 8 total, 8 up, 8 in 2024-06-18T03:15:26.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:25 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/3326824023' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2268975469"}]: dispatch 2024-06-18T03:15:26.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:25 smithi163 ceph-mon[87697]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2268975469"}]: dispatch 2024-06-18T03:15:26.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:25 smithi163 ceph-mon[82879]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-06-18T03:15:26.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:25 smithi163 ceph-mon[82879]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/3495931855"}]': finished 2024-06-18T03:15:26.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:25 smithi163 ceph-mon[82879]: osdmap e154: 8 total, 8 up, 8 in 2024-06-18T03:15:26.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:25 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/3326824023' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2268975469"}]: dispatch 2024-06-18T03:15:26.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:25 smithi163 ceph-mon[82879]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2268975469"}]: dispatch 2024-06-18T03:15:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:26 smithi186 ceph-mon[67751]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2268975469"}]': finished 2024-06-18T03:15:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:26 smithi186 ceph-mon[67751]: osdmap e155: 8 total, 8 up, 8 in 2024-06-18T03:15:26.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:26 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/198604108' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/442917201"}]: dispatch 2024-06-18T03:15:26.819 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:26 smithi186 ceph-mon[67751]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/442917201"}]: dispatch 2024-06-18T03:15:27.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:26 smithi163 ceph-mon[87697]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2268975469"}]': finished 2024-06-18T03:15:27.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:26 smithi163 ceph-mon[87697]: osdmap e155: 8 total, 8 up, 8 in 2024-06-18T03:15:27.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:26 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/198604108' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/442917201"}]: dispatch 2024-06-18T03:15:27.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:26 smithi163 ceph-mon[87697]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/442917201"}]: dispatch 2024-06-18T03:15:27.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:26 smithi163 ceph-mon[82879]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2268975469"}]': finished 2024-06-18T03:15:27.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:26 smithi163 ceph-mon[82879]: osdmap e155: 8 total, 8 up, 8 in 2024-06-18T03:15:27.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:26 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/198604108' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/442917201"}]: dispatch 2024-06-18T03:15:27.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:26 smithi163 ceph-mon[82879]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/442917201"}]: dispatch 2024-06-18T03:15:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:27 smithi186 ceph-mon[67751]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-06-18T03:15:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:27 smithi186 ceph-mon[67751]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/442917201"}]': finished 2024-06-18T03:15:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:27 smithi186 ceph-mon[67751]: osdmap e156: 8 total, 8 up, 8 in 2024-06-18T03:15:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:27 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/1659840550' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2912043813"}]: dispatch 2024-06-18T03:15:27.818 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:27 smithi186 ceph-mon[67751]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2912043813"}]: dispatch 2024-06-18T03:15:28.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:27 smithi163 ceph-mon[82879]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-06-18T03:15:28.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:27 smithi163 ceph-mon[82879]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/442917201"}]': finished 2024-06-18T03:15:28.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:27 smithi163 ceph-mon[82879]: osdmap e156: 8 total, 8 up, 8 in 2024-06-18T03:15:28.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:27 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/1659840550' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2912043813"}]: dispatch 2024-06-18T03:15:28.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:27 smithi163 ceph-mon[82879]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2912043813"}]: dispatch 2024-06-18T03:15:28.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:27 smithi163 ceph-mon[87697]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-06-18T03:15:28.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:27 smithi163 ceph-mon[87697]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/442917201"}]': finished 2024-06-18T03:15:28.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:27 smithi163 ceph-mon[87697]: osdmap e156: 8 total, 8 up, 8 in 2024-06-18T03:15:28.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:27 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/1659840550' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2912043813"}]: dispatch 2024-06-18T03:15:28.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:27 smithi163 ceph-mon[87697]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2912043813"}]: dispatch 2024-06-18T03:15:28.944 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:28 smithi163 ceph-mon[87697]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2912043813"}]': finished 2024-06-18T03:15:28.944 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:28 smithi163 ceph-mon[87697]: osdmap e157: 8 total, 8 up, 8 in 2024-06-18T03:15:28.944 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:28 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/2746845764' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4117880085"}]: dispatch 2024-06-18T03:15:28.944 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:28 smithi163 ceph-mon[87697]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4117880085"}]: dispatch 2024-06-18T03:15:28.944 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:28 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:15:28.944 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:28 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:15:28.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:28 smithi163 ceph-mon[82879]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2912043813"}]': finished 2024-06-18T03:15:28.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:28 smithi163 ceph-mon[82879]: osdmap e157: 8 total, 8 up, 8 in 2024-06-18T03:15:28.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:28 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/2746845764' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4117880085"}]: dispatch 2024-06-18T03:15:28.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:28 smithi163 ceph-mon[82879]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4117880085"}]: dispatch 2024-06-18T03:15:28.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:28 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:15:28.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:28 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:15:29.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:28 smithi186 ceph-mon[67751]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:0/2912043813"}]': finished 2024-06-18T03:15:29.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:28 smithi186 ceph-mon[67751]: osdmap e157: 8 total, 8 up, 8 in 2024-06-18T03:15:29.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:28 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/2746845764' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4117880085"}]: dispatch 2024-06-18T03:15:29.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:28 smithi186 ceph-mon[67751]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4117880085"}]: dispatch 2024-06-18T03:15:29.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:28 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:15:29.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:28 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:15:29.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:29 smithi163 ceph-mon[87697]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:15:29.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:29 smithi163 ceph-mon[87697]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4117880085"}]': finished 2024-06-18T03:15:29.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:29 smithi163 ceph-mon[87697]: osdmap e158: 8 total, 8 up, 8 in 2024-06-18T03:15:29.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:29 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/2698867013' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4117880085"}]: dispatch 2024-06-18T03:15:29.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:29 smithi163 ceph-mon[87697]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4117880085"}]: dispatch 2024-06-18T03:15:29.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:29 smithi163 ceph-mon[82879]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:15:29.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:29 smithi163 ceph-mon[82879]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4117880085"}]': finished 2024-06-18T03:15:29.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:29 smithi163 ceph-mon[82879]: osdmap e158: 8 total, 8 up, 8 in 2024-06-18T03:15:29.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:29 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/2698867013' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4117880085"}]: dispatch 2024-06-18T03:15:29.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:29 smithi163 ceph-mon[82879]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4117880085"}]: dispatch 2024-06-18T03:15:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:29 smithi186 ceph-mon[67751]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-18T03:15:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:29 smithi186 ceph-mon[67751]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6801/4117880085"}]': finished 2024-06-18T03:15:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:29 smithi186 ceph-mon[67751]: osdmap e158: 8 total, 8 up, 8 in 2024-06-18T03:15:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:29 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/2698867013' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4117880085"}]: dispatch 2024-06-18T03:15:30.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:29 smithi186 ceph-mon[67751]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4117880085"}]: dispatch 2024-06-18T03:15:30.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:30 smithi163 ceph-mon[87697]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4117880085"}]': finished 2024-06-18T03:15:30.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:30 smithi163 ceph-mon[87697]: osdmap e159: 8 total, 8 up, 8 in 2024-06-18T03:15:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:30 smithi163 ceph-mon[82879]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4117880085"}]': finished 2024-06-18T03:15:30.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:30 smithi163 ceph-mon[82879]: osdmap e159: 8 total, 8 up, 8 in 2024-06-18T03:15:31.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:30 smithi186 ceph-mon[67751]: from='client.? ' entity='client.iscsi.foo.smithi163.ucaxbw' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.163:6800/4117880085"}]': finished 2024-06-18T03:15:31.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:30 smithi186 ceph-mon[67751]: osdmap e159: 8 total, 8 up, 8 in 2024-06-18T03:15:32.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:31 smithi163 ceph-mon[87697]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:15:32.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:31 smithi163 ceph-mon[82879]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:15:32.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:31 smithi186 ceph-mon[67751]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail 2024-06-18T03:15:32.380 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:15:32.713 INFO:teuthology.orchestra.run.smithi163.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi163.ucaxbw on host smithi163 failed." 2024-06-18T03:15:32.713 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:32 smithi163 ceph-mon[87697]: from='client.44862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:32.714 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:32 smithi163 ceph-mon[82879]: from='client.44862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:33.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:32 smithi186 ceph-mon[67751]: from='client.44862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:33.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps' 2024-06-18T03:15:33.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:33 smithi163 ceph-mon[87697]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 859 B/s rd, 0 op/s 2024-06-18T03:15:33.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:33 smithi163 ceph-mon[87697]: from='client.44868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:33.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:33 smithi163 ceph-mon[87697]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:33.846 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:33 smithi163 ceph-mon[82879]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 859 B/s rd, 0 op/s 2024-06-18T03:15:33.846 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:33 smithi163 ceph-mon[82879]: from='client.44868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:33.846 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:33 smithi163 ceph-mon[82879]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:33 smithi186 ceph-mon[67751]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 859 B/s rd, 0 op/s 2024-06-18T03:15:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:33 smithi186 ceph-mon[67751]: from='client.44868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:34.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:33 smithi186 ceph-mon[67751]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:34.292 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:15:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:15:33] "GET /metrics HTTP/1.1" 200 38480 "" "Prometheus/2.43.0" 2024-06-18T03:15:35.378 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:15:35.378 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (10m) 74s ago 32m 27.1M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:15:35.378 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (9m) 74s ago 31m 79.1M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:15:35.378 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 74s ago 31m 63.4M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:15:35.378 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (11m) 74s ago 36m 444M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:15:35.378 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (21m) 74s ago 38m 507M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:15:35.378 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (8m) 74s ago 38m 61.6M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:15:35.378 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (9m) 74s ago 37m 53.1M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:15:35.378 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (7m) 74s ago 37m 49.2M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:15:35.378 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (11m) 74s ago 32m 15.5M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:15:35.379 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (11m) 74s ago 32m 15.9M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:15:35.379 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (6m) 74s ago 36m 54.4M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:15:35.379 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (6m) 74s ago 36m 55.3M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:15:35.379 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (5m) 74s ago 35m 56.1M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 05782842a9a4 2024-06-18T03:15:35.379 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (4m) 74s ago 35m 61.2M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 526499193ea3 2024-06-18T03:15:35.379 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (3m) 74s ago 34m 62.5M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 2c0ea5619d38 2024-06-18T03:15:35.379 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (3m) 74s ago 34m 60.8M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 e1d8886de2c4 2024-06-18T03:15:35.379 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (2m) 74s ago 34m 55.0M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 1bfdef63cef0 2024-06-18T03:15:35.379 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (2m) 74s ago 33m 57.5M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 a7781eaeaec5 2024-06-18T03:15:35.379 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (10m) 74s ago 33m 71.8M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:15:35.379 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (83s) 74s ago 31m 76.6M - 19.0.0-3124-g3781e3a5 af93c036e016 e5aaa8df5f66 2024-06-18T03:15:35.379 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (78s) 74s ago 31m 75.8M - 19.0.0-3124-g3781e3a5 af93c036e016 bc69579c3dbe 2024-06-18T03:15:35.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:35 smithi163 ceph-mon[82879]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 736 B/s rd, 0 op/s 2024-06-18T03:15:35.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:35 smithi163 ceph-mon[82879]: from='client.54838 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:35.900 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:35 smithi163 ceph-mon[87697]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 736 B/s rd, 0 op/s 2024-06-18T03:15:35.900 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:35 smithi163 ceph-mon[87697]: from='client.54838 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:35.924 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions' 2024-06-18T03:15:36.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:35 smithi186 ceph-mon[67751]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 736 B/s rd, 0 op/s 2024-06-18T03:15:36.070 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:35 smithi186 ceph-mon[67751]: from='client.54838 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:15:37.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:37 smithi163 ceph-mon[87697]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:37.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:37 smithi163 ceph-mon[82879]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:37.956 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:15:37.956 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:15:37.956 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:15:37.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:15:37.956 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:15:37.956 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:15:37.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:15:37.956 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:15:37.956 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 8 2024-06-18T03:15:37.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:15:37.956 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:15:37.957 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:15:37.957 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:15:37.957 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:15:37.957 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 15 2024-06-18T03:15:37.957 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:15:37.957 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:15:38.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:37 smithi186 ceph-mon[67751]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:38.629 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-06-18T03:15:38.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:38 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/218177332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:15:38.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:38 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/218177332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:15:39.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:38 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/218177332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:15:39.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:39 smithi163 ceph-mon[87697]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:15:39.861 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:39 smithi163 ceph-mon[82879]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:15:40.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:39 smithi186 ceph-mon[67751]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-18T03:15:40.348 INFO:teuthology.orchestra.run.smithi163.stdout:wait for servicemap items w/ changing names to refresh 2024-06-18T03:15:40.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'sleep 60' 2024-06-18T03:15:41.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:41 smithi163 ceph-mon[82879]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 935 B/s rd, 0 op/s 2024-06-18T03:15:41.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:41 smithi163 ceph-mon[87697]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 935 B/s rd, 0 op/s 2024-06-18T03:15:41.871 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:41 smithi186 ceph-mon[67751]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 935 B/s rd, 0 op/s 2024-06-18T03:15:42.818 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:15:42 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=cleanup t=2024-06-18T03:15:42.518301798Z level=info msg="Completed cleanup jobs" duration=3.827286ms 2024-06-18T03:15:44.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:15:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:15:43] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-06-18T03:15:44.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:43 smithi163 ceph-mon[87697]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:44.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:43 smithi163 ceph-mon[87697]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:44.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:43 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:15:44.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:43 smithi163 ceph-mon[82879]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:44.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:43 smithi163 ceph-mon[82879]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:44.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:43 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:15:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:43 smithi186 ceph-mon[67751]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:43 smithi186 ceph-mon[67751]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:44.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:43 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:15:46.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:45 smithi163 ceph-mon[87697]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:46.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:45 smithi163 ceph-mon[82879]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:46.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:45 smithi186 ceph-mon[67751]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:48.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:47 smithi163 ceph-mon[87697]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:48.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:47 smithi163 ceph-mon[82879]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:48.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:47 smithi186 ceph-mon[67751]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:50.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:49 smithi163 ceph-mon[82879]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:50.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:49 smithi163 ceph-mon[87697]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:50.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:49 smithi186 ceph-mon[67751]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:52.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:51 smithi163 ceph-mon[82879]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:52.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:51 smithi163 ceph-mon[87697]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:52.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:51 smithi186 ceph-mon[67751]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:54.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:53 smithi163 ceph-mon[87697]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:54.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:53 smithi163 ceph-mon[87697]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:54.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:53 smithi163 ceph-mon[82879]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:54.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:53 smithi163 ceph-mon[82879]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:54.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:15:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:15:53] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-06-18T03:15:54.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:53 smithi186 ceph-mon[67751]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:54.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:53 smithi186 ceph-mon[67751]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:15:56.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:55 smithi163 ceph-mon[82879]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:56.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:55 smithi163 ceph-mon[87697]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:56.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:55 smithi186 ceph-mon[67751]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:15:58.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:57 smithi163 ceph-mon[87697]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:58.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:57 smithi163 ceph-mon[82879]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:58.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:57 smithi186 ceph-mon[67751]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:15:58.944 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:58 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:15:58.944 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:58 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:15:59.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:58 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:16:00.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:15:59 smithi163 ceph-mon[87697]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:00.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:15:59 smithi163 ceph-mon[82879]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:00.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:15:59 smithi186 ceph-mon[67751]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:02.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:01 smithi163 ceph-mon[87697]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:02.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:01 smithi163 ceph-mon[82879]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:02.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:01 smithi186 ceph-mon[67751]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:04.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:03 smithi163 ceph-mon[87697]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:04.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:03 smithi163 ceph-mon[87697]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:04.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:03 smithi163 ceph-mon[82879]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:04.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:03 smithi163 ceph-mon[82879]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:04.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:16:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:16:03] "GET /metrics HTTP/1.1" 200 38483 "" "Prometheus/2.43.0" 2024-06-18T03:16:04.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:03 smithi186 ceph-mon[67751]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:04.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:03 smithi186 ceph-mon[67751]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:06.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:05 smithi163 ceph-mon[87697]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:06.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:05 smithi163 ceph-mon[82879]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:06.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:05 smithi186 ceph-mon[67751]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:08.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:07 smithi163 ceph-mon[87697]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:08.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:07 smithi163 ceph-mon[82879]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:08.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:07 smithi186 ceph-mon[67751]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:10.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:09 smithi163 ceph-mon[87697]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:10.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:09 smithi163 ceph-mon[82879]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:10.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:09 smithi186 ceph-mon[67751]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:12.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:11 smithi163 ceph-mon[87697]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:12.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:11 smithi163 ceph-mon[82879]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:12.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:11 smithi186 ceph-mon[67751]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:14.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:16:13] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-06-18T03:16:14.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[82879]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:14.043 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:16:14.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:16:14.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[82879]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 975 B/s rd, 0 op/s 2024-06-18T03:16:14.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[82879]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:14.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:16:14.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[82879]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:14.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:16:14.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[87697]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:14.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:16:14.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:16:14.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[87697]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 975 B/s rd, 0 op/s 2024-06-18T03:16:14.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[87697]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:14.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:16:14.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[87697]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:14.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:13 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:16:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:13 smithi186 ceph-mon[67751]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:14.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-18T03:16:14.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-18T03:16:14.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:13 smithi186 ceph-mon[67751]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 975 B/s rd, 0 op/s 2024-06-18T03:16:14.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:13 smithi186 ceph-mon[67751]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:14.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' 2024-06-18T03:16:14.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:13 smithi186 ceph-mon[67751]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:14.069 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:13 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:16:16.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:16 smithi163 ceph-mon[87697]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-06-18T03:16:16.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:16 smithi163 ceph-mon[82879]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-06-18T03:16:16.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:16 smithi186 ceph-mon[67751]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-06-18T03:16:18.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:18 smithi163 ceph-mon[87697]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:18.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:18 smithi163 ceph-mon[82879]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:18.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:18 smithi186 ceph-mon[67751]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:20.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:20 smithi163 ceph-mon[82879]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:20.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:20 smithi163 ceph-mon[87697]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:20.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:20 smithi186 ceph-mon[67751]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:22.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:22 smithi163 ceph-mon[82879]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:22.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:22 smithi163 ceph-mon[87697]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:22.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:22 smithi186 ceph-mon[67751]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:24.293 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:16:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:16:23] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-06-18T03:16:24.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:24 smithi163 ceph-mon[82879]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:16:24.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:24 smithi163 ceph-mon[82879]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:24.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:24 smithi163 ceph-mon[87697]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:16:24.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:24 smithi163 ceph-mon[87697]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:24 smithi186 ceph-mon[67751]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-18T03:16:24.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:24 smithi186 ceph-mon[67751]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:26.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:26 smithi163 ceph-mon[82879]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:26.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:26 smithi163 ceph-mon[87697]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:26.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:26 smithi186 ceph-mon[67751]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:28.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:28 smithi163 ceph-mon[82879]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:28.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:28 smithi163 ceph-mon[87697]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:28.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:28 smithi186 ceph-mon[67751]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:29.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:29 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:16:29.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:29 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:16:29.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:29 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:16:30.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:30 smithi163 ceph-mon[82879]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:30.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:30 smithi163 ceph-mon[87697]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:30.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:30 smithi186 ceph-mon[67751]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:32.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:32 smithi163 ceph-mon[82879]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:32.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:32 smithi163 ceph-mon[87697]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:32.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:32 smithi186 ceph-mon[67751]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:34.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:16:33 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:16:33] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-06-18T03:16:34.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:34 smithi163 ceph-mon[87697]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:34.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:34 smithi163 ceph-mon[87697]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:34.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:34 smithi163 ceph-mon[82879]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:34.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:34 smithi163 ceph-mon[82879]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:34.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:34 smithi186 ceph-mon[67751]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:34.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:34 smithi186 ceph-mon[67751]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:36.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:36 smithi163 ceph-mon[82879]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:36.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:36 smithi163 ceph-mon[87697]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:36.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:36 smithi186 ceph-mon[67751]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:38.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:38 smithi163 ceph-mon[82879]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:38.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:38 smithi163 ceph-mon[87697]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:38.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:38 smithi186 ceph-mon[67751]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:40.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:40 smithi163 ceph-mon[87697]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:40.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:40 smithi163 ceph-mon[82879]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:40.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:40 smithi186 ceph-mon[67751]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:42.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:42 smithi163 ceph-mon[87697]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:42.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:42 smithi163 ceph-mon[82879]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:42.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:42 smithi186 ceph-mon[67751]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:43.291 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ps' 2024-06-18T03:16:43.921 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:16:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:16:43] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-06-18T03:16:44.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:44 smithi163 ceph-mon[82879]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:44.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:44 smithi163 ceph-mon[82879]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:44.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:44 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:16:44.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:44 smithi163 ceph-mon[87697]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:44.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:44 smithi163 ceph-mon[87697]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:44.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:44 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:16:44.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:44 smithi186 ceph-mon[67751]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:44.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:44 smithi186 ceph-mon[67751]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:44.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:44 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:16:45.184 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-18T03:16:45.184 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (11m) 2m ago 33m 27.1M - 0.25.0 c8568f914cd2 18fe92de29b0 2024-06-18T03:16:45.184 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi186 *:3000 running (11m) 2m ago 33m 79.1M - 9.4.12 2bacad6d85d8 3fb1675854ef 2024-06-18T03:16:45.184 INFO:teuthology.orchestra.run.smithi163.stdout:iscsi.foo.smithi163.ucaxbw smithi163 error 2m ago 32m 63.4M - 3.5 e1d6a67b021e 80f63e0037b0 2024-06-18T03:16:45.184 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi186 *:8443,9283,8765 running (12m) 2m ago 37m 444M - 19.0.0-3124-g3781e3a5 af93c036e016 cf365ee8b674 2024-06-18T03:16:45.184 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (22m) 2m ago 39m 507M - 19.0.0-3124-g3781e3a5 af93c036e016 1477473a7a9c 2024-06-18T03:16:45.184 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (9m) 2m ago 39m 61.6M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 07c33ce786ba 2024-06-18T03:16:45.184 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi186 running (10m) 2m ago 38m 53.1M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 a0f80892befc 2024-06-18T03:16:45.184 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (9m) 2m ago 38m 49.2M 2048M 19.0.0-3124-g3781e3a5 af93c036e016 9950de11350d 2024-06-18T03:16:45.185 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (12m) 2m ago 34m 15.5M - 1.5.0 0da6a335fe13 6b0b89b4bc85 2024-06-18T03:16:45.185 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi186 *:9100 running (12m) 2m ago 34m 15.9M - 1.5.0 0da6a335fe13 8a17d71d6acb 2024-06-18T03:16:45.185 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (7m) 2m ago 37m 54.4M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 9442cf4db7e9 2024-06-18T03:16:45.185 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (7m) 2m ago 37m 55.3M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 63fcb171cec5 2024-06-18T03:16:45.185 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (6m) 2m ago 36m 56.1M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 05782842a9a4 2024-06-18T03:16:45.185 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (5m) 2m ago 36m 61.2M 2992M 19.0.0-3124-g3781e3a5 af93c036e016 526499193ea3 2024-06-18T03:16:45.185 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi186 running (4m) 2m ago 36m 62.5M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 2c0ea5619d38 2024-06-18T03:16:45.185 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi186 running (4m) 2m ago 35m 60.8M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 e1d8886de2c4 2024-06-18T03:16:45.185 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi186 running (3m) 2m ago 35m 55.0M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 1bfdef63cef0 2024-06-18T03:16:45.185 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi186 running (3m) 2m ago 34m 57.5M 3248M 19.0.0-3124-g3781e3a5 af93c036e016 a7781eaeaec5 2024-06-18T03:16:45.185 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi186 *:9095 running (11m) 2m ago 34m 71.8M - 2.43.0 a07b618ecd1d fd43507a2d06 2024-06-18T03:16:45.185 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi163.mvqwty smithi163 *:8000 running (2m) 2m ago 32m 76.6M - 19.0.0-3124-g3781e3a5 af93c036e016 e5aaa8df5f66 2024-06-18T03:16:45.185 INFO:teuthology.orchestra.run.smithi163.stdout:rgw.foo.smithi186.vxkxtx smithi186 *:8000 running (2m) 2m ago 32m 75.8M - 19.0.0-3124-g3781e3a5 af93c036e016 bc69579c3dbe 2024-06-18T03:16:45.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions' 2024-06-18T03:16:46.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:46 smithi163 ceph-mon[82879]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:46.379 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:46 smithi163 ceph-mon[82879]: from='client.54850 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:16:46.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:46 smithi163 ceph-mon[87697]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:46.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:46 smithi163 ceph-mon[87697]: from='client.54850 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:16:46.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:46 smithi186 ceph-mon[67751]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:46.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:46 smithi186 ceph-mon[67751]: from='client.54850 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:16:47.848 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:16:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2024-06-18T03:16:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 3 2024-06-18T03:16:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:16:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2024-06-18T03:16:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:16:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:16:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2024-06-18T03:16:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 8 2024-06-18T03:16:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:16:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw": { 2024-06-18T03:16:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 2 2024-06-18T03:16:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2024-06-18T03:16:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2024-06-18T03:16:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)": 15 2024-06-18T03:16:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: } 2024-06-18T03:16:47.850 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:16:48.453 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:48 smithi163 ceph-mon[82879]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:48.454 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:48 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/3259452948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:16:48.454 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:48 smithi163 ceph-mon[87697]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:48.454 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:48 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/3259452948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:16:48.478 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch upgrade status' 2024-06-18T03:16:48.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:48 smithi186 ceph-mon[67751]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:48.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:48 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/3259452948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:16:50.188 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:50 smithi163 ceph-mon[87697]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:50.189 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:50 smithi163 ceph-mon[82879]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:50.433 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:16:50.433 INFO:teuthology.orchestra.run.smithi163.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00", 2024-06-18T03:16:50.434 INFO:teuthology.orchestra.run.smithi163.stdout: "in_progress": true, 2024-06-18T03:16:50.434 INFO:teuthology.orchestra.run.smithi163.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-18T03:16:50.434 INFO:teuthology.orchestra.run.smithi163.stdout: "services_complete": [ 2024-06-18T03:16:50.434 INFO:teuthology.orchestra.run.smithi163.stdout: "osd", 2024-06-18T03:16:50.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr", 2024-06-18T03:16:50.434 INFO:teuthology.orchestra.run.smithi163.stdout: "mon", 2024-06-18T03:16:50.435 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw" 2024-06-18T03:16:50.435 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2024-06-18T03:16:50.435 INFO:teuthology.orchestra.run.smithi163.stdout: "progress": "15/21 daemons upgraded", 2024-06-18T03:16:50.435 INFO:teuthology.orchestra.run.smithi163.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi163.ucaxbw on host smithi163 failed.", 2024-06-18T03:16:50.435 INFO:teuthology.orchestra.run.smithi163.stdout: "is_paused": true 2024-06-18T03:16:50.435 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:16:50.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:50 smithi186 ceph-mon[67751]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:51.100 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph health detail' 2024-06-18T03:16:52.357 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:52 smithi163 ceph-mon[87697]: from='client.44898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:16:52.357 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:52 smithi163 ceph-mon[87697]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:52.358 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:52 smithi163 ceph-mon[82879]: from='client.44898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:16:52.358 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:52 smithi163 ceph-mon[82879]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:52.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:52 smithi186 ceph-mon[67751]: from='client.44898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:16:52.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:52 smithi186 ceph-mon[67751]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:53.104 INFO:teuthology.orchestra.run.smithi163.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Upgrading daemon iscsi.foo.smithi163.ucaxbw on host smithi163 failed. 2024-06-18T03:16:53.104 INFO:teuthology.orchestra.run.smithi163.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-06-18T03:16:53.104 INFO:teuthology.orchestra.run.smithi163.stdout: daemon iscsi.foo.smithi163.ucaxbw on smithi163 is in error state 2024-06-18T03:16:53.104 INFO:teuthology.orchestra.run.smithi163.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi163.ucaxbw on host smithi163 failed. 2024-06-18T03:16:53.104 INFO:teuthology.orchestra.run.smithi163.stdout: Upgrade daemon: iscsi.foo.smithi163.ucaxbw: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi163.ucaxbw ... 2024-06-18T03:16:53.104 INFO:teuthology.orchestra.run.smithi163.stdout:Creating ceph-iscsi config... 2024-06-18T03:16:53.104 INFO:teuthology.orchestra.run.smithi163.stdout:Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/iscsi-gateway.cfg 2024-06-18T03:16:53.104 INFO:teuthology.orchestra.run.smithi163.stdout:Write file: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/iscsi.foo.smithi163.ucaxbw/tcmu-runner-entrypoint.sh 2024-06-18T03:16:53.104 INFO:teuthology.orchestra.run.smithi163.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2d929affee\x2d2d1b\x2d11ef\x2dbc9d\x2dc7b262605968.slice/ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service 2024-06-18T03:16:53.104 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 1 from systemctl start ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stderr Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stderr See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout:Traceback (most recent call last): 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: return _run_code(code, main_globals, None, 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: exec(code, run_globals) 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5618, in 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 5606, in main 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1087, in deploy_daemon 2024-06-18T03:16:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/__main__.py", line 1223, in deploy_daemon_units 2024-06-18T03:16:53.106 INFO:teuthology.orchestra.run.smithi163.stdout: File "/tmp/tmp3vdsh41i.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-18T03:16:53.106 INFO:teuthology.orchestra.run.smithi163.stdout:RuntimeError: Failed command: systemctl start ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw: Job for ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service failed because the control process exited with error code. 2024-06-18T03:16:53.106 INFO:teuthology.orchestra.run.smithi163.stdout:See "systemctl status ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" and "journalctl -xeu ceph-929affee-2d1b-11ef-bc9d-c7b262605968@iscsi.foo.smithi163.ucaxbw.service" for details. 2024-06-18T03:16:53.519 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:53 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/2453711647' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:16:53.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:53 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/2453711647' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:16:53.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:53 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/2453711647' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-18T03:16:53.763 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-06-18T03:16:54.043 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:16:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:16:53] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-06-18T03:16:54.393 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:54 smithi163 ceph-mon[82879]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:54.394 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:54 smithi163 ceph-mon[82879]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:54.394 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:54 smithi163 ceph-mon[87697]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:54.394 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:54 smithi163 ceph-mon[87697]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:54.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:54 smithi186 ceph-mon[67751]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:54.569 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:54 smithi186 ceph-mon[67751]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:16:55.748 INFO:teuthology.orchestra.run.smithi163.stdout:true 2024-06-18T03:16:56.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:56 smithi163 ceph-mon[87697]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:56.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:56 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/3883765486' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:16:56.380 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:56 smithi163 ceph-mon[82879]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:56.380 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:56 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/3883765486' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:16:56.408 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-06-18T03:16:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:56 smithi186 ceph-mon[67751]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:16:56.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:56 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/3883765486' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:16:58.296 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 19.0.0-3124-g3781e3a5 (3781e3a592387d16e8020f5db99cdee4198f6d00) squid (rc)" 2024-06-18T03:16:58.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:58 smithi163 ceph-mon[87697]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:58.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:58 smithi163 ceph-mon[82879]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:58.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:58 smithi186 ceph-mon[67751]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:16:58.884 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -e sha1=3781e3a592387d16e8020f5db99cdee4198f6d00 -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-06-18T03:16:59.422 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:59 smithi163 ceph-mon[82879]: from='client.? 172.21.15.163:0/1297699930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:16:59.422 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:16:59 smithi163 ceph-mon[82879]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:16:59.422 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:59 smithi163 ceph-mon[87697]: from='client.? 172.21.15.163:0/1297699930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:16:59.422 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:16:59 smithi163 ceph-mon[87697]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:16:59.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:59 smithi186 ceph-mon[67751]: from='client.? 172.21.15.163:0/1297699930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-18T03:16:59.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:16:59 smithi186 ceph-mon[67751]: from='mgr.44106 172.21.15.163:0/1867067955' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-18T03:17:00.262 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:00 smithi163 ceph-mon[82879]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:17:00.263 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:00 smithi163 ceph-mon[87697]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:17:00.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:00 smithi186 ceph-mon[67751]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:17:00.729 INFO:teuthology.orchestra.run.smithi163.stdout:osd 8 2m ago - 2024-06-18T03:17:01.297 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-18T03:17:01.308 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi163.front.sepia.ceph.com 2024-06-18T03:17:01.308 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-06-18T03:17:02.531 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:02 smithi163 ceph-mon[82879]: from='client.54886 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:17:02.531 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:02 smithi163 ceph-mon[82879]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:17:02.532 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:02 smithi163 ceph-mon[87697]: from='client.54886 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:17:02.532 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:02 smithi163 ceph-mon[87697]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:17:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:02 smithi186 ceph-mon[67751]: from='client.54886 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:17:02.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:02 smithi186 ceph-mon[67751]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:17:03.910 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2024-06-18T03:17:03.911 INFO:teuthology.orchestra.run.smithi163.stdout: "image": "quay.io/ceph/ceph", 2024-06-18T03:17:03.911 INFO:teuthology.orchestra.run.smithi163.stdout: "registry": "quay.io", 2024-06-18T03:17:03.911 INFO:teuthology.orchestra.run.smithi163.stdout: "bare_image": "ceph/ceph", 2024-06-18T03:17:03.911 INFO:teuthology.orchestra.run.smithi163.stdout: "versions": [] 2024-06-18T03:17:03.911 INFO:teuthology.orchestra.run.smithi163.stdout:} 2024-06-18T03:17:04.172 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:17:03] "GET /metrics HTTP/1.1" 200 38483 "" "Prometheus/2.43.0" 2024-06-18T03:17:04.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:04 smithi163 ceph-mon[87697]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:17:04.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:04 smithi163 ceph-mon[87697]: from='client.44928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:17:04.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:04 smithi163 ceph-mon[87697]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:17:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:04 smithi163 ceph-mon[82879]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:17:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:04 smithi163 ceph-mon[82879]: from='client.44928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:17:04.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:04 smithi163 ceph-mon[82879]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:17:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:04 smithi186 ceph-mon[67751]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:17:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:04 smithi186 ceph-mon[67751]: from='client.44928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:17:04.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:04 smithi186 ceph-mon[67751]: from='client.54775 -' entity='client.iscsi.foo.smithi163.ucaxbw' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-18T03:17:04.633 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-06-18T03:17:06.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:06 smithi163 ceph-mon[82879]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:17:06.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:06 smithi163 ceph-mon[87697]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:17:06.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:06 smithi186 ceph-mon[67751]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:17:07.225 INFO:teuthology.orchestra.run.smithi163.stdout: "16.2.0", 2024-06-18T03:17:07.855 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-06-18T03:17:08.162 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:07 smithi163 ceph-mon[87697]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:17:08.162 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:07 smithi163 ceph-mon[87697]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:17:08.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:07 smithi163 ceph-mon[82879]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:17:08.163 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:07 smithi163 ceph-mon[82879]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:17:08.318 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:07 smithi186 ceph-mon[67751]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:17:08.319 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:07 smithi186 ceph-mon[67751]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-18T03:17:10.468 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:10 smithi163 ceph-mon[87697]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:17:10.468 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:10 smithi163 ceph-mon[82879]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:17:10.468 INFO:teuthology.orchestra.run.smithi163.stdout: "v16.2.2", 2024-06-18T03:17:10.468 INFO:teuthology.orchestra.run.smithi163.stdout: "v16.2.2-20210505", 2024-06-18T03:17:10.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:10 smithi186 ceph-mon[67751]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-18T03:17:11.111 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-06-18T03:17:11.120 INFO:tasks.cephadm:Teardown begin 2024-06-18T03:17:11.120 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-18T03:17:11.148 DEBUG:teuthology.orchestra.run.smithi186:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-18T03:17:11.182 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-06-18T03:17:11.182 DEBUG:teuthology.orchestra.run.smithi163:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-06-18T03:17:11.204 DEBUG:teuthology.orchestra.run.smithi186:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-06-18T03:17:11.239 INFO:tasks.cephadm:Stopping all daemons... 2024-06-18T03:17:11.240 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-06-18T03:17:11.240 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.a 2024-06-18T03:17:11.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:11 smithi163 ceph-mon[87697]: from='client.44940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:17:11.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:11 smithi163 ceph-mon[82879]: from='client.44940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:17:11.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:11 smithi163 systemd[1]: Stopping Ceph mon.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:17:11.568 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:11 smithi186 ceph-mon[67751]: from='client.44940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-18T03:17:11.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a[82874]: 2024-06-18T03:17:11.545+0000 7f7f37d1c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:17:11.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a[82874]: 2024-06-18T03:17:11.545+0000 7f7f37d1c640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-06-18T03:17:11.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:11 smithi163 podman[128305]: 2024-06-18 03:17:11.566127413 +0000 UTC m=+0.125597843 container died 07c33ce786ba407922a5e24ccec20cedb7b1c687edf7795de0bf4ab41b14f6a9 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-3781e3a, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:17:11.863 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:17:11] ENGINE Bus STOPPING 2024-06-18T03:17:12.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:11 smithi163 podman[128305]: 2024-06-18 03:17:11.953746931 +0000 UTC m=+0.513217352 container cleanup 07c33ce786ba407922a5e24ccec20cedb7b1c687edf7795de0bf4ab41b14f6a9 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, RELEASE=squid-3781e3a) 2024-06-18T03:17:12.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Jun 18 03:17:11 smithi163 bash[128305]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-a 2024-06-18T03:17:12.206 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:17:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-18T03:17:12.206 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:17:11] ENGINE Bus STOPPED 2024-06-18T03:17:12.206 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:17:11] ENGINE Bus STARTING 2024-06-18T03:17:12.206 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:17:11] ENGINE Serving on http://:::9283 2024-06-18T03:17:12.206 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:11 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: [18/Jun/2024:03:17:11] ENGINE Bus STARTED 2024-06-18T03:17:12.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.a.service' 2024-06-18T03:17:12.421 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:17:12.422 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-06-18T03:17:12.422 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-06-18T03:17:12.422 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.c 2024-06-18T03:17:12.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:12 smithi163 systemd[1]: Stopping Ceph mon.c for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:17:13.167 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:12 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c[87693]: 2024-06-18T03:17:12.791+0000 7f23081d6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:17:13.167 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:12 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c[87693]: 2024-06-18T03:17:12.791+0000 7f23081d6640 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-06-18T03:17:13.167 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:12 smithi163 podman[128439]: 2024-06-18 03:17:12.914247333 +0000 UTC m=+0.261023645 container died 9950de11350dd7ede23a5bc68fe602e24120cab1c49148a1b541bfda46aae0b2 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux ) 2024-06-18T03:17:13.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:13 smithi163 podman[128439]: 2024-06-18 03:17:13.291652165 +0000 UTC m=+0.638428520 container cleanup 9950de11350dd7ede23a5bc68fe602e24120cab1c49148a1b541bfda46aae0b2 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-18T03:17:13.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Jun 18 03:17:13 smithi163 bash[128439]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-c 2024-06-18T03:17:13.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.c.service' 2024-06-18T03:17:13.746 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:17:13.746 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-06-18T03:17:13.746 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-06-18T03:17:13.746 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.b 2024-06-18T03:17:14.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:13 smithi186 systemd[1]: Stopping Ceph mon.b for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:17:14.293 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y[62740]: ::ffff:172.21.15.186 - - [18/Jun/2024:03:17:13] "GET /metrics HTTP/1.1" 200 38480 "" "Prometheus/2.43.0" 2024-06-18T03:17:14.340 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b[67747]: 2024-06-18T03:17:14.084+0000 7f24ece98640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:17:14.340 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b[67747]: 2024-06-18T03:17:14.084+0000 7f24ece98640 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-06-18T03:17:14.635 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:14 smithi186 podman[102225]: 2024-06-18 03:17:14.340194758 +0000 UTC m=+0.386580163 container died a0f80892befcff66556062ba6dc843e01acd9d2bba7557f65833e262968f9f1e (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-3781e3a) 2024-06-18T03:17:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:14 smithi186 podman[102225]: 2024-06-18 03:17:14.810447893 +0000 UTC m=+0.856833337 container cleanup a0f80892befcff66556062ba6dc843e01acd9d2bba7557f65833e262968f9f1e (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-3781e3a, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:17:15.068 INFO:journalctl@ceph.mon.b.smithi186.stdout:Jun 18 03:17:14 smithi186 bash[102225]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mon-b 2024-06-18T03:17:15.235 DEBUG:teuthology.orchestra.run.smithi186:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mon.b.service' 2024-06-18T03:17:15.283 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:17:15.283 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-06-18T03:17:15.283 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-06-18T03:17:15.283 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.y 2024-06-18T03:17:15.618 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:15 smithi163 systemd[1]: Stopping Ceph mgr.y for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:17:15.621 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:15 smithi163 podman[128560]: 2024-06-18 03:17:15.618786351 +0000 UTC m=+0.137794260 container died 1477473a7a9cb55c495620bbe881eb7047dd72a45de5415d5ba4754f9968bd69 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, RELEASE=squid-3781e3a, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-06-18T03:17:16.256 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:16 smithi163 podman[128560]: 2024-06-18 03:17:16.054589427 +0000 UTC m=+0.573597331 container cleanup 1477473a7a9cb55c495620bbe881eb7047dd72a45de5415d5ba4754f9968bd69 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, RELEASE=squid-3781e3a, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-06-18T03:17:16.256 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:16 smithi163 bash[128560]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y 2024-06-18T03:17:16.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.y.service' 2024-06-18T03:17:16.517 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:16 smithi163 podman[128571]: 2024-06-18 03:17:16.256504622 +0000 UTC m=+0.629721176 container remove 1477473a7a9cb55c495620bbe881eb7047dd72a45de5415d5ba4754f9968bd69 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-y, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a, GIT_CLEAN=True) 2024-06-18T03:17:16.517 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:16 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.y.service: Deactivated successfully. 2024-06-18T03:17:16.517 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:16 smithi163 systemd[1]: Stopped Ceph mgr.y for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:17:16.517 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Jun 18 03:17:16 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.y.service: Consumed 41.900s CPU time. 2024-06-18T03:17:16.530 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:17:16.530 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-06-18T03:17:16.531 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-06-18T03:17:16.531 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.x 2024-06-18T03:17:16.818 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:17:16 smithi186 systemd[1]: Stopping Ceph mgr.x for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:17:17.318 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:17:16 smithi186 podman[102346]: 2024-06-18 03:17:16.871337578 +0000 UTC m=+0.139885254 container died cf365ee8b674129940913305ed6e62988ebd190a3a4fe27e13a2dc3fb11e9691 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_CLEAN=True) 2024-06-18T03:17:17.609 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:17:17 smithi186 podman[102346]: 2024-06-18 03:17:17.323988871 +0000 UTC m=+0.592536585 container cleanup cf365ee8b674129940913305ed6e62988ebd190a3a4fe27e13a2dc3fb11e9691 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:17:17.609 INFO:journalctl@ceph.mgr.x.smithi186.stdout:Jun 18 03:17:17 smithi186 bash[102346]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-mgr-x 2024-06-18T03:17:17.755 DEBUG:teuthology.orchestra.run.smithi186:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@mgr.x.service' 2024-06-18T03:17:17.810 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:17:17.811 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-06-18T03:17:17.811 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-06-18T03:17:17.811 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.0 2024-06-18T03:17:18.133 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:17 smithi163 systemd[1]: Stopping Ceph osd.0 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:17:18.542 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:18 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0[94575]: 2024-06-18T03:17:18.133+0000 7f83f2fb0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:17:18.543 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:18 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0[94575]: 2024-06-18T03:17:18.133+0000 7f83f2fb0640 -1 osd.0 159 *** Got signal Terminated *** 2024-06-18T03:17:18.543 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:18 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0[94575]: 2024-06-18T03:17:18.133+0000 7f83f2fb0640 -1 osd.0 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:17:23.900 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:23 smithi163 podman[128683]: 2024-06-18 03:17:23.622088862 +0000 UTC m=+5.614570188 container died 9442cf4db7e9d068c139a3c526ba85bb123b2379af854cfdc127badc13ac1f55 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-3781e3a, ceph=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:17:23.901 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:23 smithi163 podman[128683]: 2024-06-18 03:17:23.900539918 +0000 UTC m=+5.893021234 container cleanup 9442cf4db7e9d068c139a3c526ba85bb123b2379af854cfdc127badc13ac1f55 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-3781e3a, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:17:24.159 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:23 smithi163 bash[128683]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0 2024-06-18T03:17:24.528 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:24 smithi163 podman[128701]: 2024-06-18 03:17:24.160106749 +0000 UTC m=+0.538157704 container remove 9442cf4db7e9d068c139a3c526ba85bb123b2379af854cfdc127badc13ac1f55 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD) 2024-06-18T03:17:24.529 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:24 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:24.280Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:17:24.529 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:24 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:24.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:17:24.793 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:24 smithi163 podman[128788]: 2024-06-18 03:17:24.430877436 +0000 UTC m=+0.017234134 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:17:24.793 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:24 smithi163 podman[128788]: 2024-06-18 03:17:24.544425869 +0000 UTC m=+0.130782604 container create fa3b8b18e73414d43f9c515007e8caae18c5ecbc84cf53d7dd502c6de2cb4abd (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, ceph=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-3781e3a) 2024-06-18T03:17:24.793 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:24 smithi163 podman[128788]: 2024-06-18 03:17:24.738665261 +0000 UTC m=+0.325021955 container init fa3b8b18e73414d43f9c515007e8caae18c5ecbc84cf53d7dd502c6de2cb4abd (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0) 2024-06-18T03:17:24.793 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:24 smithi163 podman[128788]: 2024-06-18 03:17:24.741987721 +0000 UTC m=+0.328344411 container start fa3b8b18e73414d43f9c515007e8caae18c5ecbc84cf53d7dd502c6de2cb4abd (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-06-18T03:17:25.246 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:24 smithi163 podman[128788]: 2024-06-18 03:17:24.821924934 +0000 UTC m=+0.408281643 container attach fa3b8b18e73414d43f9c515007e8caae18c5ecbc84cf53d7dd502c6de2cb4abd (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-06-18T03:17:25.246 INFO:journalctl@ceph.osd.0.smithi163.stdout:Jun 18 03:17:25 smithi163 podman[128788]: 2024-06-18 03:17:25.003507299 +0000 UTC m=+0.589863991 container died fa3b8b18e73414d43f9c515007e8caae18c5ecbc84cf53d7dd502c6de2cb4abd (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:17:25.661 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.0.service' 2024-06-18T03:17:25.701 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:17:25.701 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-06-18T03:17:25.701 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-06-18T03:17:25.701 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.1 2024-06-18T03:17:26.187 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:25 smithi163 systemd[1]: Stopping Ceph osd.1 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:17:26.543 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1[99711]: 2024-06-18T03:17:26.186+0000 7f0006297640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:17:26.543 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1[99711]: 2024-06-18T03:17:26.186+0000 7f0006297640 -1 osd.1 159 *** Got signal Terminated *** 2024-06-18T03:17:26.543 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:26 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1[99711]: 2024-06-18T03:17:26.186+0000 7f0006297640 -1 osd.1 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:17:31.987 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:31 smithi163 podman[128901]: 2024-06-18 03:17:31.717853031 +0000 UTC m=+5.649103641 container died 63fcb171cec5509e202886a2c361abc65bb8c8b140d62466fe1513be2f1b806b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-06-18T03:17:32.293 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:32 smithi163 podman[128901]: 2024-06-18 03:17:32.095487681 +0000 UTC m=+6.026738286 container cleanup 63fcb171cec5509e202886a2c361abc65bb8c8b140d62466fe1513be2f1b806b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240612, io.buildah.version=1.36.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:17:32.293 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:32 smithi163 bash[128901]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1 2024-06-18T03:17:32.714 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:32 smithi163 podman[128919]: 2024-06-18 03:17:32.371882897 +0000 UTC m=+0.651042189 container remove 63fcb171cec5509e202886a2c361abc65bb8c8b140d62466fe1513be2f1b806b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:17:32.983 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:32 smithi163 podman[129006]: 2024-06-18 03:17:32.714355647 +0000 UTC m=+0.106695086 container create b78ddc53917ac81ced2ad551ae2ea4d833e29bcf1694a536a7d90f4ec7ecfe07 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-06-18T03:17:32.983 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:32 smithi163 podman[129006]: 2024-06-18 03:17:32.624586756 +0000 UTC m=+0.016926203 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:17:32.983 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:32 smithi163 podman[129006]: 2024-06-18 03:17:32.908455462 +0000 UTC m=+0.300794905 container init b78ddc53917ac81ced2ad551ae2ea4d833e29bcf1694a536a7d90f4ec7ecfe07 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-06-18T03:17:32.983 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:32 smithi163 podman[129006]: 2024-06-18 03:17:32.911569177 +0000 UTC m=+0.303908604 container start b78ddc53917ac81ced2ad551ae2ea4d833e29bcf1694a536a7d90f4ec7ecfe07 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-deactivate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:17:32.983 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:32 smithi163 podman[129006]: 2024-06-18 03:17:32.983254345 +0000 UTC m=+0.375593785 container attach b78ddc53917ac81ced2ad551ae2ea4d833e29bcf1694a536a7d90f4ec7ecfe07 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-deactivate, io.buildah.version=1.36.0, GIT_CLEAN=True, RELEASE=squid-3781e3a, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, CEPH_POINT_RELEASE=) 2024-06-18T03:17:33.293 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:33 smithi163 podman[129006]: 2024-06-18 03:17:33.170371999 +0000 UTC m=+0.562711440 container died b78ddc53917ac81ced2ad551ae2ea4d833e29bcf1694a536a7d90f4ec7ecfe07 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3781e3a) 2024-06-18T03:17:33.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.1.service' 2024-06-18T03:17:33.755 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:33 smithi163 podman[129028]: 2024-06-18 03:17:33.712035805 +0000 UTC m=+0.532474639 container remove b78ddc53917ac81ced2ad551ae2ea4d833e29bcf1694a536a7d90f4ec7ecfe07 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-1-deactivate, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a) 2024-06-18T03:17:33.756 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:33 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.1.service: Deactivated successfully. 2024-06-18T03:17:33.756 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:33 smithi163 systemd[1]: Stopped Ceph osd.1 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:17:33.756 INFO:journalctl@ceph.osd.1.smithi163.stdout:Jun 18 03:17:33 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.1.service: Consumed 6.353s CPU time. 2024-06-18T03:17:33.771 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:17:33.771 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-06-18T03:17:33.771 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-06-18T03:17:33.771 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.2 2024-06-18T03:17:34.042 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:33 smithi163 systemd[1]: Stopping Ceph osd.2 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:17:34.543 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:34 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2[105987]: 2024-06-18T03:17:34.139+0000 7f4ba30cb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:17:34.543 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:34 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2[105987]: 2024-06-18T03:17:34.139+0000 7f4ba30cb640 -1 osd.2 159 *** Got signal Terminated *** 2024-06-18T03:17:34.543 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:34 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2[105987]: 2024-06-18T03:17:34.139+0000 7f4ba30cb640 -1 osd.2 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:17:34.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:34 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:34.280Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:17:34.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:34 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:34.280Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:17:34.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:34 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:34.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:17:39.849 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:39 smithi163 podman[129107]: 2024-06-18 03:17:39.575429497 +0000 UTC m=+5.570567123 container died 05782842a9a41071ab86edecaaca7da717f8f4c226a1f2befbfb9a1eb37cfdfc (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-06-18T03:17:40.190 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:39 smithi163 podman[129107]: 2024-06-18 03:17:39.939535126 +0000 UTC m=+5.934672751 container cleanup 05782842a9a41071ab86edecaaca7da717f8f4c226a1f2befbfb9a1eb37cfdfc (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:17:40.190 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:39 smithi163 bash[129107]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2 2024-06-18T03:17:40.482 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:40 smithi163 podman[129121]: 2024-06-18 03:17:40.190634128 +0000 UTC m=+0.613233199 container remove 05782842a9a41071ab86edecaaca7da717f8f4c226a1f2befbfb9a1eb37cfdfc (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-3781e3a, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:17:40.760 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:40 smithi163 podman[129214]: 2024-06-18 03:17:40.483108379 +0000 UTC m=+0.085322002 container create a0b5e61a78718edf6e76adbbe5e57ee66dce73a644f75f94db32cdce8d3985bb (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-deactivate, RELEASE=squid-3781e3a, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-06-18T03:17:40.760 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:40 smithi163 podman[129214]: 2024-06-18 03:17:40.414851218 +0000 UTC m=+0.017064839 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:17:40.760 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:40 smithi163 podman[129214]: 2024-06-18 03:17:40.693533866 +0000 UTC m=+0.295747493 container init a0b5e61a78718edf6e76adbbe5e57ee66dce73a644f75f94db32cdce8d3985bb (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a) 2024-06-18T03:17:40.761 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:40 smithi163 podman[129214]: 2024-06-18 03:17:40.6967552 +0000 UTC m=+0.298968806 container start a0b5e61a78718edf6e76adbbe5e57ee66dce73a644f75f94db32cdce8d3985bb (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-deactivate, RELEASE=squid-3781e3a, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-06-18T03:17:40.761 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:40 smithi163 podman[129214]: 2024-06-18 03:17:40.760684279 +0000 UTC m=+0.362897900 container attach a0b5e61a78718edf6e76adbbe5e57ee66dce73a644f75f94db32cdce8d3985bb (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-deactivate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-06-18T03:17:41.043 INFO:journalctl@ceph.osd.2.smithi163.stdout:Jun 18 03:17:40 smithi163 podman[129214]: 2024-06-18 03:17:40.956354662 +0000 UTC m=+0.558568281 container died a0b5e61a78718edf6e76adbbe5e57ee66dce73a644f75f94db32cdce8d3985bb (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0) 2024-06-18T03:17:41.505 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.2.service' 2024-06-18T03:17:41.544 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:17:41.544 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-06-18T03:17:41.544 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-06-18T03:17:41.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.3 2024-06-18T03:17:41.792 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:41 smithi163 systemd[1]: Stopping Ceph osd.3 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:17:42.293 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:41 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3[112415]: 2024-06-18T03:17:41.882+0000 7f5f88b13640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:17:42.293 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:41 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3[112415]: 2024-06-18T03:17:41.882+0000 7f5f88b13640 -1 osd.3 159 *** Got signal Terminated *** 2024-06-18T03:17:42.293 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:41 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3[112415]: 2024-06-18T03:17:41.882+0000 7f5f88b13640 -1 osd.3 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:17:44.043 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:43.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:17:44.043 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:43 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:43.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:17:44.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:44 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:44.281Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:17:44.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:44 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:44.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:17:44.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:44 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:44.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:17:45.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:45 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:45.279+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:45.319 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:45 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:45.059+0000 7f1164e40640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:21.547645+0000 front 2024-06-18T03:17:21.547562+0000 (oldest deadline 2024-06-18T03:17:44.447459+0000) 2024-06-18T03:17:46.485 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:46 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:46.103+0000 7f1164e40640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:21.547645+0000 front 2024-06-18T03:17:21.547562+0000 (oldest deadline 2024-06-18T03:17:44.447459+0000) 2024-06-18T03:17:46.486 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:46 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:46.241+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:47.396 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:47 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:47.128+0000 7f1164e40640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:21.547645+0000 front 2024-06-18T03:17:21.547562+0000 (oldest deadline 2024-06-18T03:17:44.447459+0000) 2024-06-18T03:17:47.396 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:47 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:47.213+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:47.659 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:47 smithi163 podman[129316]: 2024-06-18 03:17:47.398950402 +0000 UTC m=+5.634620971 container died 526499193ea3c0b3d7d0921b84721ef025a3efe76cd7e1a010f62d7b5c775548 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS) 2024-06-18T03:17:47.909 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:47 smithi163 podman[129316]: 2024-06-18 03:17:47.741744384 +0000 UTC m=+5.977414938 container cleanup 526499193ea3c0b3d7d0921b84721ef025a3efe76cd7e1a010f62d7b5c775548 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, ceph=True, GIT_CLEAN=True) 2024-06-18T03:17:47.910 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:47 smithi163 bash[129316]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3 2024-06-18T03:17:48.292 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:47 smithi163 podman[129334]: 2024-06-18 03:17:47.909848833 +0000 UTC m=+0.505105678 container remove 526499193ea3c0b3d7d0921b84721ef025a3efe76cd7e1a010f62d7b5c775548 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, GIT_CLEAN=True, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=) 2024-06-18T03:17:48.518 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:48.086+0000 7f1164e40640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:21.547645+0000 front 2024-06-18T03:17:21.547562+0000 (oldest deadline 2024-06-18T03:17:44.447459+0000) 2024-06-18T03:17:48.518 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:48.188+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:48.687 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:48 smithi163 podman[129420]: 2024-06-18 03:17:48.263632065 +0000 UTC m=+0.017510099 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:17:48.687 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:48 smithi163 podman[129420]: 2024-06-18 03:17:48.394061151 +0000 UTC m=+0.147939184 container create be4a91b4e768a8a8a977e5672eb2c5375073cad16634a622834e154d02dab52b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:17:48.688 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:48 smithi163 podman[129420]: 2024-06-18 03:17:48.604333373 +0000 UTC m=+0.358211418 container init be4a91b4e768a8a8a977e5672eb2c5375073cad16634a622834e154d02dab52b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-06-18T03:17:48.688 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:48 smithi163 podman[129420]: 2024-06-18 03:17:48.607670328 +0000 UTC m=+0.361548361 container start be4a91b4e768a8a8a977e5672eb2c5375073cad16634a622834e154d02dab52b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, org.label-schema.build-date=20240612, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-06-18T03:17:48.688 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:48 smithi163 podman[129420]: 2024-06-18 03:17:48.687853118 +0000 UTC m=+0.441731163 container attach be4a91b4e768a8a8a977e5672eb2c5375073cad16634a622834e154d02dab52b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, ceph=True, org.label-schema.schema-version=1.0) 2024-06-18T03:17:48.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:48 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:48.517+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:17:49.042 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:48 smithi163 podman[129420]: 2024-06-18 03:17:48.868953849 +0000 UTC m=+0.622831893 container died be4a91b4e768a8a8a977e5672eb2c5375073cad16634a622834e154d02dab52b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240612, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-06-18T03:17:49.318 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:49.062+0000 7f1164e40640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:21.547645+0000 front 2024-06-18T03:17:21.547562+0000 (oldest deadline 2024-06-18T03:17:44.447459+0000) 2024-06-18T03:17:49.319 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:49.219+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:49.375 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.3.service' 2024-06-18T03:17:49.399 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:49 smithi163 podman[129443]: 2024-06-18 03:17:49.349491992 +0000 UTC m=+0.474746391 container remove be4a91b4e768a8a8a977e5672eb2c5375073cad16634a622834e154d02dab52b (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=) 2024-06-18T03:17:49.399 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:49 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.3.service: Deactivated successfully. 2024-06-18T03:17:49.399 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:49 smithi163 systemd[1]: Stopped Ceph osd.3 for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:17:49.399 INFO:journalctl@ceph.osd.3.smithi163.stdout:Jun 18 03:17:49 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.3.service: Consumed 6.246s CPU time. 2024-06-18T03:17:49.413 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:17:49.414 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-06-18T03:17:49.414 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-06-18T03:17:49.414 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.4 2024-06-18T03:17:49.818 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:49 smithi186 systemd[1]: Stopping Ceph osd.4 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:17:49.819 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:49.478+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:17:49.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:49.542+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:17:50.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:50 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:50.172+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:50.318 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:49.976+0000 7f116c832640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:17:50.319 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:49.976+0000 7f116c832640 -1 osd.4 159 *** Got signal Terminated *** 2024-06-18T03:17:50.319 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:49 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:49.976+0000 7f116c832640 -1 osd.4 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:17:50.319 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:50 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:50.068+0000 7f1164e40640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:21.547645+0000 front 2024-06-18T03:17:21.547562+0000 (oldest deadline 2024-06-18T03:17:44.447459+0000) 2024-06-18T03:17:50.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:50 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:50.501+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:17:50.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:50 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:50.578+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:17:51.318 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:51.063+0000 7f1164e40640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:21.547645+0000 front 2024-06-18T03:17:21.547562+0000 (oldest deadline 2024-06-18T03:17:44.447459+0000) 2024-06-18T03:17:51.319 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:51.155+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:51.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:51.546+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:17:51.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:51 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:51.617+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:17:52.499 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:52.090+0000 7f1164e40640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:21.547645+0000 front 2024-06-18T03:17:21.547562+0000 (oldest deadline 2024-06-18T03:17:44.447459+0000) 2024-06-18T03:17:52.500 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:52.115+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:52.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:52.596+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:17:52.819 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:52.499+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:17:52.819 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:52.499+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:17:53.528 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:53.115+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:53.529 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:53.100+0000 7f1164e40640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:21.547645+0000 front 2024-06-18T03:17:21.547562+0000 (oldest deadline 2024-06-18T03:17:44.447459+0000) 2024-06-18T03:17:53.529 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:53.100+0000 7f1164e40640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.348383+0000 front 2024-06-18T03:17:30.348449+0000 (oldest deadline 2024-06-18T03:17:52.549082+0000) 2024-06-18T03:17:53.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:53.528+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:17:53.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:53.528+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:17:53.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:53.565+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:17:53.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:53 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:53.565+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:17:54.043 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:53.555Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=5 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:17:54.043 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:53.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:17:54.043 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:53 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:53.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:17:54.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:54 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:54.281Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:17:54.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:54 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:54.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:17:54.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:17:54 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:17:54.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:17:54.545 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:54 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:54.080+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:54.545 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:54 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:54.135+0000 7f1164e40640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:21.547645+0000 front 2024-06-18T03:17:21.547562+0000 (oldest deadline 2024-06-18T03:17:44.447459+0000) 2024-06-18T03:17:54.545 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:54 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4[78617]: 2024-06-18T03:17:54.135+0000 7f1164e40640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.348383+0000 front 2024-06-18T03:17:30.348449+0000 (oldest deadline 2024-06-18T03:17:52.549082+0000) 2024-06-18T03:17:54.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:54 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:54.544+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:17:54.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:54 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:54.544+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:17:54.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:54 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:54.564+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:17:54.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:54 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:54.564+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:17:55.476 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:55.069+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:55.477 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:55.069+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:29.616791+0000 front 2024-06-18T03:17:29.616736+0000 (oldest deadline 2024-06-18T03:17:54.316642+0000) 2024-06-18T03:17:55.744 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:55 smithi186 podman[102482]: 2024-06-18 03:17:55.476153269 +0000 UTC m=+5.862029044 container died 2c0ea5619d3880176bd7e31b04f022f26452351b124c02be7514bd8ea200b737 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, ceph=True, RELEASE=squid-3781e3a, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-18T03:17:55.744 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:55.581+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:17:55.745 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:55.581+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:17:55.745 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:55.547+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:17:55.745 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:55 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:55.547+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:17:56.046 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:55 smithi186 podman[102482]: 2024-06-18 03:17:55.918309382 +0000 UTC m=+6.304185151 container cleanup 2c0ea5619d3880176bd7e31b04f022f26452351b124c02be7514bd8ea200b737 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:17:56.046 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:55 smithi186 bash[102482]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4 2024-06-18T03:17:56.318 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:56 smithi186 podman[102497]: 2024-06-18 03:17:56.128537473 +0000 UTC m=+0.648190022 container remove 2c0ea5619d3880176bd7e31b04f022f26452351b124c02be7514bd8ea200b737 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD) 2024-06-18T03:17:56.319 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:56.045+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:56.319 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:56.045+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:29.616791+0000 front 2024-06-18T03:17:29.616736+0000 (oldest deadline 2024-06-18T03:17:54.316642+0000) 2024-06-18T03:17:56.319 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:17:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:17:56.232Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.163:8765: connect: connection refused" 2024-06-18T03:17:56.319 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:17:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:17:56.235Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.163:8765: connect: connection refused" 2024-06-18T03:17:56.319 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:17:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:17:56.235Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.163:8765: connect: connection refused" 2024-06-18T03:17:56.319 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:17:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:17:56.237Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.163:8765: connect: connection refused" 2024-06-18T03:17:56.319 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:17:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:17:56.237Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.163:8765: connect: connection refused" 2024-06-18T03:17:56.782 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:56.590+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:17:56.782 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:56.590+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:17:56.782 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:56.577+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:17:56.782 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:56 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:56.577+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:17:56.782 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:56 smithi186 podman[102585]: 2024-06-18 03:17:56.529486613 +0000 UTC m=+0.102784992 container create b74ac4ce39ad8d01dae1409102baca111bf3a811351ae829399124fa3dd03108 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-3781e3a, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240612) 2024-06-18T03:17:56.783 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:56 smithi186 podman[102585]: 2024-06-18 03:17:56.443874383 +0000 UTC m=+0.017172720 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:17:57.061 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:56 smithi186 podman[102585]: 2024-06-18 03:17:56.782208268 +0000 UTC m=+0.355506613 container init b74ac4ce39ad8d01dae1409102baca111bf3a811351ae829399124fa3dd03108 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-deactivate, RELEASE=squid-3781e3a, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-06-18T03:17:57.061 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:56 smithi186 podman[102585]: 2024-06-18 03:17:56.786259203 +0000 UTC m=+0.359557547 container start b74ac4ce39ad8d01dae1409102baca111bf3a811351ae829399124fa3dd03108 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:17:57.061 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:56 smithi186 podman[102585]: 2024-06-18 03:17:56.873820538 +0000 UTC m=+0.447118888 container attach b74ac4ce39ad8d01dae1409102baca111bf3a811351ae829399124fa3dd03108 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240612, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, ceph=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:17:57.318 INFO:journalctl@ceph.osd.4.smithi186.stdout:Jun 18 03:17:57 smithi186 podman[102585]: 2024-06-18 03:17:57.061873256 +0000 UTC m=+0.635171601 container died b74ac4ce39ad8d01dae1409102baca111bf3a811351ae829399124fa3dd03108 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-06-18T03:17:57.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:57.093+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:57.319 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:57.093+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:29.616791+0000 front 2024-06-18T03:17:29.616736+0000 (oldest deadline 2024-06-18T03:17:54.316642+0000) 2024-06-18T03:17:57.611 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:57.575+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:17:57.611 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:57.575+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:17:57.665 DEBUG:teuthology.orchestra.run.smithi186:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.4.service' 2024-06-18T03:17:57.706 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:17:57.707 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-06-18T03:17:57.707 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-06-18T03:17:57.707 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.5 2024-06-18T03:17:58.048 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:57.610+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:17:58.048 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:57 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:57.610+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:17:58.048 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:57 smithi186 systemd[1]: Stopping Ceph osd.5 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:17:58.049 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:58.047+0000 7fa5b4769640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:17:58.049 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:58.047+0000 7fa5b4769640 -1 osd.5 159 *** Got signal Terminated *** 2024-06-18T03:17:58.049 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:58.047+0000 7fa5b4769640 -1 osd.5 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:17:58.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:58.082+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:58.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:58.082+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:29.616791+0000 front 2024-06-18T03:17:29.616736+0000 (oldest deadline 2024-06-18T03:17:54.316642+0000) 2024-06-18T03:17:59.034 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:58.569+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:17:59.034 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:58.569+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:17:59.034 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:58.577+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:17:59.034 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:58 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:58.577+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:17:59.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:59.034+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:17:59.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:17:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:17:59.034+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:29.616791+0000 front 2024-06-18T03:17:29.616736+0000 (oldest deadline 2024-06-18T03:17:54.316642+0000) 2024-06-18T03:17:59.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:59.549+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:17:59.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:17:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:17:59.549+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:17:59.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:59.522+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:17:59.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:17:59 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:17:59.522+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:00.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:18:00.018+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:18:00.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:18:00.018+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:29.616791+0000 front 2024-06-18T03:17:29.616736+0000 (oldest deadline 2024-06-18T03:17:54.316642+0000) 2024-06-18T03:18:00.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:00.579+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:18:00.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:00.579+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:18:00.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:00.489+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:00.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:00.489+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:01.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:18:00.978+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:18:01.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:00 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:18:00.978+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:29.616791+0000 front 2024-06-18T03:17:29.616736+0000 (oldest deadline 2024-06-18T03:17:54.316642+0000) 2024-06-18T03:18:01.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:01.595+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:18:01.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:01.595+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:18:01.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:01.492+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:01.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:01.492+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:02.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:18:01.976+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:18:02.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:01 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:18:01.976+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:29.616791+0000 front 2024-06-18T03:17:29.616736+0000 (oldest deadline 2024-06-18T03:17:54.316642+0000) 2024-06-18T03:18:02.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:02.617+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:18:02.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:02.617+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:18:02.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:02.617+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:36.683830+0000 front 2024-06-18T03:17:36.683815+0000 (oldest deadline 2024-06-18T03:18:02.583541+0000) 2024-06-18T03:18:02.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:02.454+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:02.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:02.454+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:03.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:18:02.943+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:19.314563+0000 front 2024-06-18T03:17:19.314576+0000 (oldest deadline 2024-06-18T03:17:44.614465+0000) 2024-06-18T03:18:03.318 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:02 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5[83540]: 2024-06-18T03:18:02.943+0000 7fa5acd77640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:29.616791+0000 front 2024-06-18T03:17:29.616736+0000 (oldest deadline 2024-06-18T03:17:54.316642+0000) 2024-06-18T03:18:03.731 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:03 smithi186 podman[102692]: 2024-06-18 03:18:03.451678769 +0000 UTC m=+5.513623379 container died e1d8886de2c4d14df083a15d31794138f4be51ebacfd845221d0d996e57647d1 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=) 2024-06-18T03:18:03.732 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:03.601+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:18:03.732 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:03.601+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:18:03.732 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:03.601+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:36.683830+0000 front 2024-06-18T03:17:36.683815+0000 (oldest deadline 2024-06-18T03:18:02.583541+0000) 2024-06-18T03:18:03.732 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:03.439+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:03.732 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:03.439+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:03.732 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:03 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:03.439+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:04.042 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:03.556Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=5 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:18:04.043 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:03.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:18:04.043 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:03 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:03.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:18:04.068 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:03 smithi186 podman[102692]: 2024-06-18 03:18:03.922529419 +0000 UTC m=+5.984474068 container cleanup e1d8886de2c4d14df083a15d31794138f4be51ebacfd845221d0d996e57647d1 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:18:04.068 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:03 smithi186 bash[102692]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5 2024-06-18T03:18:04.461 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:04 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:04.282Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:18:04.461 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:04 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:04.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:18:04.461 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:04 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:04.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:18:04.568 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:04 smithi186 podman[102710]: 2024-06-18 03:18:04.318254837 +0000 UTC m=+0.863065578 container remove e1d8886de2c4d14df083a15d31794138f4be51ebacfd845221d0d996e57647d1 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-18T03:18:04.568 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:04 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:04.448+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:04.568 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:04 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:04.448+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:04.568 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:04 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:04.448+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:04.920 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:04 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:04.627+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:18:04.920 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:04 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:04.627+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:18:04.920 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:04 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:04.627+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:36.683830+0000 front 2024-06-18T03:17:36.683815+0000 (oldest deadline 2024-06-18T03:18:02.583541+0000) 2024-06-18T03:18:04.920 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:04 smithi186 podman[102798]: 2024-06-18 03:18:04.612949217 +0000 UTC m=+0.016955718 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:18:04.920 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:04 smithi186 podman[102798]: 2024-06-18 03:18:04.72581892 +0000 UTC m=+0.129825464 container create 8f14f6635ac97c89c57b2d85eaea143ad705c692714c11c4b4fba7634c8d7134 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3781e3a, io.buildah.version=1.36.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-06-18T03:18:04.920 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:04 smithi186 podman[102798]: 2024-06-18 03:18:04.919760778 +0000 UTC m=+0.323767277 container init 8f14f6635ac97c89c57b2d85eaea143ad705c692714c11c4b4fba7634c8d7134 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240612, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:18:05.181 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:04 smithi186 podman[102798]: 2024-06-18 03:18:04.923260849 +0000 UTC m=+0.327267356 container start 8f14f6635ac97c89c57b2d85eaea143ad705c692714c11c4b4fba7634c8d7134 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-3781e3a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-06-18T03:18:05.181 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:04 smithi186 podman[102798]: 2024-06-18 03:18:04.994680905 +0000 UTC m=+0.398687423 container attach 8f14f6635ac97c89c57b2d85eaea143ad705c692714c11c4b4fba7634c8d7134 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-deactivate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, io.buildah.version=1.36.0) 2024-06-18T03:18:05.181 INFO:journalctl@ceph.osd.5.smithi186.stdout:Jun 18 03:18:05 smithi186 podman[102798]: 2024-06-18 03:18:05.181363991 +0000 UTC m=+0.585370492 container died 8f14f6635ac97c89c57b2d85eaea143ad705c692714c11c4b4fba7634c8d7134 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-5-deactivate, RELEASE=squid-3781e3a, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True) 2024-06-18T03:18:05.472 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:05 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:05.424+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:05.473 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:05 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:05.424+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:05.473 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:05 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:05.424+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:05.765 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:05 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:05.659+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:18:05.765 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:05 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:05.659+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:18:05.765 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:05 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:05.659+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:36.683830+0000 front 2024-06-18T03:17:36.683815+0000 (oldest deadline 2024-06-18T03:18:02.583541+0000) 2024-06-18T03:18:05.793 DEBUG:teuthology.orchestra.run.smithi186:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.5.service' 2024-06-18T03:18:05.833 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:18:05.833 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-06-18T03:18:05.834 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-06-18T03:18:05.834 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.6 2024-06-18T03:18:06.068 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:05 smithi186 systemd[1]: Stopping Ceph osd.6 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:18:06.471 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:06.167+0000 7f99a84a3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:18:06.471 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:06.167+0000 7f99a84a3640 -1 osd.6 159 *** Got signal Terminated *** 2024-06-18T03:18:06.471 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:06.167+0000 7f99a84a3640 -1 osd.6 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:18:06.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:06.652+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:18:06.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:06.652+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:18:06.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:06.652+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:36.683830+0000 front 2024-06-18T03:17:36.683815+0000 (oldest deadline 2024-06-18T03:18:02.583541+0000) 2024-06-18T03:18:06.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:06.471+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:06.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:06.471+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:06.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:06 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:06.471+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:07.604 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:07.442+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:07.605 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:07.442+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:07.605 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:07.442+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:07.913 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:07.604+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:18:07.913 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:07.604+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:18:07.913 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:07 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:07.604+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:36.683830+0000 front 2024-06-18T03:17:36.683815+0000 (oldest deadline 2024-06-18T03:18:02.583541+0000) 2024-06-18T03:18:08.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:08.595+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:18:08.819 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:08.595+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:18:08.819 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:08.595+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:36.683830+0000 front 2024-06-18T03:17:36.683815+0000 (oldest deadline 2024-06-18T03:18:02.583541+0000) 2024-06-18T03:18:08.819 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:08.595+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2024-06-18T03:17:42.583987+0000 front 2024-06-18T03:17:42.583970+0000 (oldest deadline 2024-06-18T03:18:08.484014+0000) 2024-06-18T03:18:08.820 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:08.468+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:08.820 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:08.468+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:08.820 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:08 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:08.468+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:09.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:09.601+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:18:09.819 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:09.601+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:18:09.819 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:09.601+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:36.683830+0000 front 2024-06-18T03:17:36.683815+0000 (oldest deadline 2024-06-18T03:18:02.583541+0000) 2024-06-18T03:18:09.819 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:09.601+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2024-06-18T03:17:42.583987+0000 front 2024-06-18T03:17:42.583970+0000 (oldest deadline 2024-06-18T03:18:08.484014+0000) 2024-06-18T03:18:09.820 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:09.493+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:09.820 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:09.493+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:09.820 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:09.493+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:09.820 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:09 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:09.493+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2024-06-18T03:17:43.152877+0000 front 2024-06-18T03:17:43.152960+0000 (oldest deadline 2024-06-18T03:18:09.052869+0000) 2024-06-18T03:18:10.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:10 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:10.496+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:10.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:10 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:10.496+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:10.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:10 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:10.496+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:10.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:10 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:10.496+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2024-06-18T03:17:43.152877+0000 front 2024-06-18T03:17:43.152960+0000 (oldest deadline 2024-06-18T03:18:09.052869+0000) 2024-06-18T03:18:10.819 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:10 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:10.616+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:18.581580+0000 front 2024-06-18T03:17:18.581464+0000 (oldest deadline 2024-06-18T03:17:47.981674+0000) 2024-06-18T03:18:10.819 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:10 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:10.616+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:30.282944+0000 front 2024-06-18T03:17:30.282922+0000 (oldest deadline 2024-06-18T03:17:51.982648+0000) 2024-06-18T03:18:10.819 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:10 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:10.616+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:36.683830+0000 front 2024-06-18T03:17:36.683815+0000 (oldest deadline 2024-06-18T03:18:02.583541+0000) 2024-06-18T03:18:10.819 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:10 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6[88460]: 2024-06-18T03:18:10.616+0000 7f99a0ab1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2024-06-18T03:17:42.583987+0000 front 2024-06-18T03:17:42.583970+0000 (oldest deadline 2024-06-18T03:18:08.484014+0000) 2024-06-18T03:18:11.818 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:11 smithi186 podman[102901]: 2024-06-18 03:18:11.533521342 +0000 UTC m=+5.478377945 container died 1bfdef63cef0453b2d0006cfd96fcb5a4134365a98146b36c2bbe5abf6986403 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-3781e3a, ceph=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240612, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-06-18T03:18:11.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:11.541+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:11.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:11.541+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:11.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:11.541+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:11.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:11 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:11.541+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2024-06-18T03:17:43.152877+0000 front 2024-06-18T03:17:43.152960+0000 (oldest deadline 2024-06-18T03:18:09.052869+0000) 2024-06-18T03:18:12.210 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:11 smithi186 podman[102901]: 2024-06-18 03:18:11.909316967 +0000 UTC m=+5.854173616 container cleanup 1bfdef63cef0453b2d0006cfd96fcb5a4134365a98146b36c2bbe5abf6986403 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.36.0, RELEASE=squid-3781e3a, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-18T03:18:12.210 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:11 smithi186 bash[102901]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6 2024-06-18T03:18:12.512 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:12 smithi186 podman[102916]: 2024-06-18 03:18:12.210508207 +0000 UTC m=+0.672762986 container remove 1bfdef63cef0453b2d0006cfd96fcb5a4134365a98146b36c2bbe5abf6986403 (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.36.0) 2024-06-18T03:18:12.780 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:12 smithi186 podman[103007]: 2024-06-18 03:18:12.438374045 +0000 UTC m=+0.017737018 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:18:12.780 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:12 smithi186 podman[103007]: 2024-06-18 03:18:12.563543446 +0000 UTC m=+0.142906447 container create f735b71c3c04cfdc99844a39a5326cd014325675522002573cdefebdb27d78ae (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-deactivate, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240612, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a) 2024-06-18T03:18:12.780 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:12.512+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:12.780 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:12.512+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:12.781 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:12.512+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:12.781 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:12 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:12.512+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2024-06-18T03:17:43.152877+0000 front 2024-06-18T03:17:43.152960+0000 (oldest deadline 2024-06-18T03:18:09.052869+0000) 2024-06-18T03:18:13.036 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:12 smithi186 podman[103007]: 2024-06-18 03:18:12.780435194 +0000 UTC m=+0.359798167 container init f735b71c3c04cfdc99844a39a5326cd014325675522002573cdefebdb27d78ae (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240612, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-3781e3a, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True) 2024-06-18T03:18:13.037 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:12 smithi186 podman[103007]: 2024-06-18 03:18:12.784062821 +0000 UTC m=+0.363425781 container start f735b71c3c04cfdc99844a39a5326cd014325675522002573cdefebdb27d78ae (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-deactivate, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a, org.label-schema.license=GPLv2, org.label-schema.build-date=20240612, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:18:13.037 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:12 smithi186 podman[103007]: 2024-06-18 03:18:12.880716951 +0000 UTC m=+0.460079913 container attach f735b71c3c04cfdc99844a39a5326cd014325675522002573cdefebdb27d78ae (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3781e3a, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=) 2024-06-18T03:18:13.318 INFO:journalctl@ceph.osd.6.smithi186.stdout:Jun 18 03:18:13 smithi186 podman[103007]: 2024-06-18 03:18:13.037019689 +0000 UTC m=+0.616382649 container died f735b71c3c04cfdc99844a39a5326cd014325675522002573cdefebdb27d78ae (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240612) 2024-06-18T03:18:13.575 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:13.529+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:13.575 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:13.529+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:13.575 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:13.529+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:13.576 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:13.529+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2024-06-18T03:17:43.152877+0000 front 2024-06-18T03:17:43.152960+0000 (oldest deadline 2024-06-18T03:18:09.052869+0000) 2024-06-18T03:18:13.597 DEBUG:teuthology.orchestra.run.smithi186:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.6.service' 2024-06-18T03:18:13.635 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:18:13.636 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-06-18T03:18:13.636 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-06-18T03:18:13.636 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.7 2024-06-18T03:18:13.962 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:13 smithi186 systemd[1]: Stopping Ceph osd.7 for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:18:14.043 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:13.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:18:14.043 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:13.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:18:14.043 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:13 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:13.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:18:14.318 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:13.961+0000 7f1d382ca640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-18T03:18:14.318 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:13.961+0000 7f1d382ca640 -1 osd.7 159 *** Got signal Terminated *** 2024-06-18T03:18:14.318 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:13 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:13.961+0000 7f1d382ca640 -1 osd.7 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-18T03:18:14.542 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:14 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:14.282Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:18:14.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:14 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:14.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:18:14.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:14 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:14.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:18:14.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:14.520+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:14.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:14.520+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:14.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:14.520+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:14.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:14 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:14.520+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2024-06-18T03:17:43.152877+0000 front 2024-06-18T03:17:43.152960+0000 (oldest deadline 2024-06-18T03:18:09.052869+0000) 2024-06-18T03:18:15.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:15.567+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:15.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:15.567+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:15.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:15.567+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:15.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:15 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:15.567+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2024-06-18T03:17:43.152877+0000 front 2024-06-18T03:17:43.152960+0000 (oldest deadline 2024-06-18T03:18:09.052869+0000) 2024-06-18T03:18:16.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:16 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:16.554+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:16.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:16 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:16.554+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:16.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:16 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:16.554+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:16.818 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:16 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:16.554+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2024-06-18T03:17:43.152877+0000 front 2024-06-18T03:17:43.152960+0000 (oldest deadline 2024-06-18T03:18:09.052869+0000) 2024-06-18T03:18:16.819 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:16 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:16.554+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.186:6804 osd.4 since back 2024-06-18T03:17:51.754753+0000 front 2024-06-18T03:17:51.754865+0000 (oldest deadline 2024-06-18T03:18:15.854750+0000) 2024-06-18T03:18:17.874 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:17.601+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:17.874 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:17.601+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:17.874 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:17.601+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:17.874 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:17.601+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2024-06-18T03:17:43.152877+0000 front 2024-06-18T03:17:43.152960+0000 (oldest deadline 2024-06-18T03:18:09.052869+0000) 2024-06-18T03:18:17.874 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:17 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:17.601+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.186:6804 osd.4 since back 2024-06-18T03:17:51.754753+0000 front 2024-06-18T03:17:51.754865+0000 (oldest deadline 2024-06-18T03:18:15.854750+0000) 2024-06-18T03:18:19.068 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:18.629+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2024-06-18T03:17:17.350335+0000 front 2024-06-18T03:17:17.350147+0000 (oldest deadline 2024-06-18T03:17:48.550564+0000) 2024-06-18T03:18:19.068 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:18.629+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2024-06-18T03:17:28.551188+0000 front 2024-06-18T03:17:28.551229+0000 (oldest deadline 2024-06-18T03:17:53.151376+0000) 2024-06-18T03:18:19.069 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:18.629+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2024-06-18T03:17:37.852352+0000 front 2024-06-18T03:17:37.852340+0000 (oldest deadline 2024-06-18T03:18:03.152316+0000) 2024-06-18T03:18:19.069 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:18.629+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2024-06-18T03:17:43.152877+0000 front 2024-06-18T03:17:43.152960+0000 (oldest deadline 2024-06-18T03:18:09.052869+0000) 2024-06-18T03:18:19.069 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:18 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7[92952]: 2024-06-18T03:18:18.629+0000 7f1d308d8640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.186:6804 osd.4 since back 2024-06-18T03:17:51.754753+0000 front 2024-06-18T03:17:51.754865+0000 (oldest deadline 2024-06-18T03:18:15.854750+0000) 2024-06-18T03:18:19.697 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:19 smithi186 podman[103108]: 2024-06-18 03:18:19.424739742 +0000 UTC m=+5.563973202 container died a7781eaeaec54baf2978aa94b21100fddbd74a3ebb13d40da15c8c472b849f3a (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7, org.label-schema.license=GPLv2, RELEASE=squid-3781e3a, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux ) 2024-06-18T03:18:20.029 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:19 smithi186 podman[103108]: 2024-06-18 03:18:19.795170801 +0000 UTC m=+5.934404261 container cleanup a7781eaeaec54baf2978aa94b21100fddbd74a3ebb13d40da15c8c472b849f3a (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0) 2024-06-18T03:18:20.029 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:19 smithi186 bash[103108]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7 2024-06-18T03:18:20.318 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:20 smithi186 podman[103136]: 2024-06-18 03:18:20.029726655 +0000 UTC m=+0.602890994 container remove a7781eaeaec54baf2978aa94b21100fddbd74a3ebb13d40da15c8c472b849f3a (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240612, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-3781e3a) 2024-06-18T03:18:20.691 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:20 smithi186 podman[103224]: 2024-06-18 03:18:20.36404712 +0000 UTC m=+0.110906918 container create 4e66c38b5e9a704dccbc32bd3a3d91e39c6385b7a6dd6fe7fadc896f0eea7d1d (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240612, RELEASE=squid-3781e3a) 2024-06-18T03:18:20.692 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:20 smithi186 podman[103224]: 2024-06-18 03:18:20.271502218 +0000 UTC m=+0.018361974 image pull af93c036e0161a7a6f4fcb3f1f48acaac5cf4ef8953c6a8b104629a4b86fec44 quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00 2024-06-18T03:18:20.692 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:20 smithi186 podman[103224]: 2024-06-18 03:18:20.599707581 +0000 UTC m=+0.346567340 container init 4e66c38b5e9a704dccbc32bd3a3d91e39c6385b7a6dd6fe7fadc896f0eea7d1d (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-deactivate, RELEASE=squid-3781e3a, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240612, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-06-18T03:18:20.692 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:20 smithi186 podman[103224]: 2024-06-18 03:18:20.602878795 +0000 UTC m=+0.349738548 container start 4e66c38b5e9a704dccbc32bd3a3d91e39c6385b7a6dd6fe7fadc896f0eea7d1d (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3781e3a, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612) 2024-06-18T03:18:20.692 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:20 smithi186 podman[103224]: 2024-06-18 03:18:20.691778621 +0000 UTC m=+0.438638387 container attach 4e66c38b5e9a704dccbc32bd3a3d91e39c6385b7a6dd6fe7fadc896f0eea7d1d (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240612, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-3781e3a, ceph=True) 2024-06-18T03:18:21.068 INFO:journalctl@ceph.osd.7.smithi186.stdout:Jun 18 03:18:20 smithi186 podman[103224]: 2024-06-18 03:18:20.862256106 +0000 UTC m=+0.609115869 container died 4e66c38b5e9a704dccbc32bd3a3d91e39c6385b7a6dd6fe7fadc896f0eea7d1d (image=quay.ceph.io/ceph-ci/ceph:3781e3a592387d16e8020f5db99cdee4198f6d00, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.build-date=20240612, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-3781e3a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-18T03:18:21.541 DEBUG:teuthology.orchestra.run.smithi186:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@osd.7.service' 2024-06-18T03:18:21.583 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:18:21.583 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-06-18T03:18:21.583 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-06-18T03:18:21.583 DEBUG:teuthology.orchestra.run.smithi186:> sudo systemctl stop ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a 2024-06-18T03:18:21.818 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 systemd[1]: Stopping Ceph prometheus.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:18:22.190 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:18:21.914Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-18T03:18:22.190 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:18:21.915Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-06-18T03:18:22.190 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:18:21.915Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-06-18T03:18:22.190 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:18:21.915Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-18T03:18:22.190 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:18:21.915Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-06-18T03:18:22.190 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:18:21.915Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-06-18T03:18:22.191 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:18:21.915Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-06-18T03:18:22.191 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:18:21.915Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-06-18T03:18:22.191 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:18:21.915Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-06-18T03:18:22.191 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:18:21.916Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-06-18T03:18:22.191 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:18:21.916Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-06-18T03:18:22.191 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a[63961]: ts=2024-06-18T03:18:21.916Z caller=main.go:1147 level=info msg="See you next time!" 2024-06-18T03:18:22.191 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:21 smithi186 podman[103325]: 2024-06-18 03:18:21.92529281 +0000 UTC m=+0.120942336 container died fd43507a2d0669bf42d13b9a126dda2eeed1e7d8ebf60ddfe2b7a6687724ad21 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:18:22.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:22 smithi186 podman[103325]: 2024-06-18 03:18:22.381686065 +0000 UTC m=+0.577335589 container cleanup fd43507a2d0669bf42d13b9a126dda2eeed1e7d8ebf60ddfe2b7a6687724ad21 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-18T03:18:22.568 INFO:journalctl@ceph.prometheus.a.smithi186.stdout:Jun 18 03:18:22 smithi186 bash[103325]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-prometheus-a 2024-06-18T03:18:22.804 DEBUG:teuthology.orchestra.run.smithi186:> sudo pkill -f 'journalctl -f -n 0 -u ceph-929affee-2d1b-11ef-bc9d-c7b262605968@prometheus.a.service' 2024-06-18T03:18:22.837 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-18T03:18:22.837 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-06-18T03:18:22.838 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 --force --keep-logs 2024-06-18T03:18:23.964 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:23.558Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:18:23.965 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:23.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:18:23.965 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:23 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:23.559Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:18:24.543 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:18:24 smithi163 systemd[1]: Stopping Ceph node-exporter.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:18:24.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:24 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:24.283Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:18:24.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:24 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:24.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi163.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.163:8443: connect: connection refused" 2024-06-18T03:18:24.544 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:24 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:24.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi186.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.186:8443: connect: connection refused" 2024-06-18T03:18:24.832 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:18:24 smithi163 podman[129811]: 2024-06-18 03:18:24.560391365 +0000 UTC m=+0.122609532 container died 6b0b89b4bc8567b6b4b05ab39e6a6580b34ae4d089cb4d479682c290dc85693f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T03:18:25.283 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:18:25 smithi163 podman[129811]: 2024-06-18 03:18:25.002054094 +0000 UTC m=+0.564272254 container cleanup 6b0b89b4bc8567b6b4b05ab39e6a6580b34ae4d089cb4d479682c290dc85693f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T03:18:25.283 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:18:25 smithi163 bash[129811]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a 2024-06-18T03:18:25.542 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:18:25 smithi163 podman[129823]: 2024-06-18 03:18:25.28356495 +0000 UTC m=+0.727117489 container remove 6b0b89b4bc8567b6b4b05ab39e6a6580b34ae4d089cb4d479682c290dc85693f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-18T03:18:25.543 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:18:25 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-06-18T03:18:25.543 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:18:25 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-06-18T03:18:25.543 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:18:25 smithi163 systemd[1]: Stopped Ceph node-exporter.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:18:25.543 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Jun 18 03:18:25 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.a.service: Consumed 3.070s CPU time. 2024-06-18T03:18:25.902 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:25 smithi163 systemd[1]: Stopping Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:18:25.902 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:25 smithi163 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a[78111]: ts=2024-06-18T03:18:25.861Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-06-18T03:18:26.203 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:25 smithi163 podman[129929]: 2024-06-18 03:18:25.902617152 +0000 UTC m=+0.146641674 container died 18fe92de29b081c82c028060524c3d79dfbb13c8995fedad0d037a0fa2d20206 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T03:18:26.203 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:26 smithi163 podman[129929]: 2024-06-18 03:18:26.203348093 +0000 UTC m=+0.447372613 container cleanup 18fe92de29b081c82c028060524c3d79dfbb13c8995fedad0d037a0fa2d20206 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T03:18:26.542 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:26 smithi163 bash[129929]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a 2024-06-18T03:18:26.543 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:26 smithi163 podman[129942]: 2024-06-18 03:18:26.429843493 +0000 UTC m=+0.530267094 container remove 18fe92de29b081c82c028060524c3d79dfbb13c8995fedad0d037a0fa2d20206 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-18T03:18:26.851 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:26 smithi163 podman[129942]: 2024-06-18 03:18:26.547112614 +0000 UTC m=+0.647536217 volume remove 10a06faa0e0061e630000a281694b371f385a2012070fc753935adabe69b47c4 2024-06-18T03:18:26.851 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:26 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-06-18T03:18:26.851 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:26 smithi163 systemd[1]: Stopped Ceph alertmanager.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:18:26.851 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Jun 18 03:18:26 smithi163 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@alertmanager.a.service: Consumed 1.650s CPU time. 2024-06-18T03:18:49.568 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 --force --keep-logs 2024-06-18T03:18:51.318 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:18:51 smithi186 systemd[1]: Stopping Ceph node-exporter.b for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:18:51.692 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:18:51 smithi186 podman[103736]: 2024-06-18 03:18:51.427849272 +0000 UTC m=+0.112894920 container died 8a17d71d6acb9520cf6e136cd705fe45e9fdc7414dad15a2bf12476af8316a5a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T03:18:52.043 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:18:51 smithi186 podman[103736]: 2024-06-18 03:18:51.867503666 +0000 UTC m=+0.552549360 container cleanup 8a17d71d6acb9520cf6e136cd705fe45e9fdc7414dad15a2bf12476af8316a5a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T03:18:52.043 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:18:51 smithi186 bash[103736]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b 2024-06-18T03:18:52.303 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:18:52 smithi186 podman[103749]: 2024-06-18 03:18:52.04357785 +0000 UTC m=+0.614988921 container remove 8a17d71d6acb9520cf6e136cd705fe45e9fdc7414dad15a2bf12476af8316a5a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-18T03:18:52.303 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:18:52 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-06-18T03:18:52.568 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:18:52 smithi186 systemd[1]: Stopping Ceph grafana.a for 929affee-2d1b-11ef-bc9d-c7b262605968... 2024-06-18T03:18:52.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:18:52 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-06-18T03:18:52.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:18:52 smithi186 systemd[1]: Stopped Ceph node-exporter.b for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:18:52.569 INFO:journalctl@ceph.node-exporter.b.smithi186.stdout:Jun 18 03:18:52 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@node-exporter.b.service: Consumed 3.060s CPU time. 2024-06-18T03:18:53.021 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:18:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=server t=2024-06-18T03:18:52.721877974Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-06-18T03:18:53.021 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:18:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=tracing t=2024-06-18T03:18:52.722246735Z level=info msg="Closing tracing" 2024-06-18T03:18:53.021 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:18:52 smithi186 ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a[66172]: logger=ticker t=2024-06-18T03:18:52.722274844Z level=info msg=stopped last_tick=2024-06-18T03:18:50Z 2024-06-18T03:18:53.021 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:18:52 smithi186 podman[103858]: 2024-06-18 03:18:52.73208849 +0000 UTC m=+0.106308377 container died 3fb1675854efe7724486edb1e728588521ac1cdcea6d4325db8732c9d4096d3d (image=quay.io/ceph/grafana:9.4.12, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-06-18T03:18:53.021 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:18:53 smithi186 podman[103858]: 2024-06-18 03:18:53.021353732 +0000 UTC m=+0.395573616 container cleanup 3fb1675854efe7724486edb1e728588521ac1cdcea6d4325db8732c9d4096d3d (image=quay.io/ceph/grafana:9.4.12, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-06-18T03:18:53.021 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:18:53 smithi186 bash[103858]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a 2024-06-18T03:18:53.318 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:18:53 smithi186 podman[103872]: 2024-06-18 03:18:53.247551985 +0000 UTC m=+0.517020467 container remove 3fb1675854efe7724486edb1e728588521ac1cdcea6d4325db8732c9d4096d3d (image=quay.io/ceph/grafana:9.4.12, name=ceph-929affee-2d1b-11ef-bc9d-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-06-18T03:18:53.800 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:18:53 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@grafana.a.service: Deactivated successfully. 2024-06-18T03:18:53.801 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:18:53 smithi186 systemd[1]: Stopped Ceph grafana.a for 929affee-2d1b-11ef-bc9d-c7b262605968. 2024-06-18T03:18:53.801 INFO:journalctl@ceph.grafana.a.smithi186.stdout:Jun 18 03:18:53 smithi186 systemd[1]: ceph-929affee-2d1b-11ef-bc9d-c7b262605968@grafana.a.service: Consumed 10.675s CPU time. 2024-06-18T03:19:05.013 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-18T03:19:05.047 DEBUG:teuthology.orchestra.run.smithi186:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-18T03:19:05.074 INFO:tasks.cephadm:Archiving crash dumps... 2024-06-18T03:19:05.076 DEBUG:teuthology.misc:Transferring archived files from smithi163:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-06-17_21:08:03-orch-squid-distro-default-smithi/7760446/remote/smithi163/crash 2024-06-18T03:19:05.077 DEBUG:teuthology.orchestra.run.smithi163:> sudo tar c -f - -C /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/crash -- . 2024-06-18T03:19:05.125 INFO:teuthology.orchestra.run.smithi163.stderr:tar: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/crash: Cannot open: No such file or directory 2024-06-18T03:19:05.126 INFO:teuthology.orchestra.run.smithi163.stderr:tar: Error is not recoverable: exiting now 2024-06-18T03:19:05.127 DEBUG:teuthology.misc:Transferring archived files from smithi186:/var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-06-17_21:08:03-orch-squid-distro-default-smithi/7760446/remote/smithi186/crash 2024-06-18T03:19:05.128 DEBUG:teuthology.orchestra.run.smithi186:> sudo tar c -f - -C /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/crash -- . 2024-06-18T03:19:05.182 INFO:teuthology.orchestra.run.smithi186.stderr:tar: /var/lib/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/crash: Cannot open: No such file or directory 2024-06-18T03:19:05.182 INFO:teuthology.orchestra.run.smithi186.stderr:tar: Error is not recoverable: exiting now 2024-06-18T03:19:05.183 INFO:tasks.cephadm:Checking cluster log for badness... 2024-06-18T03:19:05.183 DEBUG:teuthology.orchestra.run.smithi163:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-18T03:19:05.213 INFO:teuthology.orchestra.run.smithi163.stdout:2024-06-18T03:04:03.531297+0000 mon.a (mon.0) 1086 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-06-18T03:19:05.214 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-06-18T03:19:05.215 DEBUG:teuthology.orchestra.run.smithi163:> sudo egrep '\[SEC\]' /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-18T03:19:05.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo egrep '\[ERR\]' /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-18T03:19:05.354 DEBUG:teuthology.orchestra.run.smithi163:> sudo egrep '\[WRN\]' /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-18T03:19:05.422 INFO:teuthology.orchestra.run.smithi163.stdout:2024-06-18T03:04:03.531297+0000 mon.a (mon.0) 1086 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-06-18T03:19:05.422 INFO:tasks.cephadm:Compressing logs... 2024-06-18T03:19:05.422 DEBUG:teuthology.orchestra.run.smithi163:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-06-18T03:19:05.464 DEBUG:teuthology.orchestra.run.smithi186:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-06-18T03:19:05.488 INFO:teuthology.orchestra.run.smithi163.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.log 2024-06-18T03:19:05.489 INFO:teuthology.orchestra.run.smithi163.stderr:: No such file or directory 2024-06-18T03:19:05.489 INFO:teuthology.orchestra.run.smithi163.stderr:gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.2.log 2024-06-18T03:19:05.489 INFO:teuthology.orchestra.run.smithi163.stderr:gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.audit.log 2024-06-18T03:19:05.489 INFO:teuthology.orchestra.run.smithi163.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.log: /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.3.log 2024-06-18T03:19:05.489 INFO:teuthology.orchestra.run.smithi163.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-client.rgw.foo.smithi163.mvqwty.log 2024-06-18T03:19:05.493 INFO:teuthology.orchestra.run.smithi163.stderr:gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.1.log 2024-06-18T03:19:05.493 INFO:teuthology.orchestra.run.smithi163.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.3.log: /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-client.rgw.foo.smithi163.mvqwty.log: gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mgr.y.log 2024-06-18T03:19:05.493 INFO:teuthology.orchestra.run.smithi163.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.1.log: 76.3% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-client.rgw.foo.smithi163.mvqwty.log.gz 2024-06-18T03:19:05.493 INFO:teuthology.orchestra.run.smithi163.stderr:gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.0.log 2024-06-18T03:19:05.493 INFO:teuthology.orchestra.run.smithi163.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mon.c.log 2024-06-18T03:19:05.493 INFO:teuthology.orchestra.run.smithi163.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mon.a.log 2024-06-18T03:19:05.496 INFO:teuthology.orchestra.run.smithi163.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mon.c.log: 94.0% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.log.gz 2024-06-18T03:19:05.496 INFO:teuthology.orchestra.run.smithi163.stderr:gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.cephadm.log 2024-06-18T03:19:05.499 INFO:teuthology.orchestra.run.smithi186.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.log 2024-06-18T03:19:05.499 INFO:teuthology.orchestra.run.smithi186.stderr:: No such file or directory 2024-06-18T03:19:05.499 INFO:teuthology.orchestra.run.smithi186.stderr:gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.audit.log 2024-06-18T03:19:05.499 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.7.log 2024-06-18T03:19:05.499 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.5.log 2024-06-18T03:19:05.499 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.4.log 2024-06-18T03:19:05.499 INFO:teuthology.orchestra.run.smithi186.stderr:gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mgr.x.log 2024-06-18T03:19:05.499 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.5.log: /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.6.log 2024-06-18T03:19:05.499 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.cephadm.log 2024-06-18T03:19:05.499 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-client.rgw.foo.smithi186.vxkxtx.log 2024-06-18T03:19:05.499 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.cephadm.log: 90.0% 89.1% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.log.gz 2024-06-18T03:19:05.499 INFO:teuthology.orchestra.run.smithi186.stderr: -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.cephadm.log.gz 2024-06-18T03:19:05.500 INFO:teuthology.orchestra.run.smithi186.stderr:gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-volume.log 2024-06-18T03:19:05.500 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-client.rgw.foo.smithi186.vxkxtx.log: gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mon.b.log 2024-06-18T03:19:05.500 INFO:teuthology.orchestra.run.smithi186.stderr: 75.9% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-client.rgw.foo.smithi186.vxkxtx.log.gz 2024-06-18T03:19:05.500 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-06-18T03:19:05.500 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mon.b.log: 91.1% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.audit.log.gz 2024-06-18T03:19:05.500 INFO:teuthology.orchestra.run.smithi163.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mon.a.log: 94.5% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.audit.log.gz 2024-06-18T03:19:05.509 INFO:teuthology.orchestra.run.smithi163.stderr:gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/tcmu-runner.log 2024-06-18T03:19:05.511 INFO:teuthology.orchestra.run.smithi186.stderr:/var/log/ceph/cephadm.log: 91.0% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-volume.log.gz 2024-06-18T03:19:05.512 INFO:teuthology.orchestra.run.smithi163.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.cephadm.log: 94.1% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph.cephadm.log.gz 2024-06-18T03:19:05.522 INFO:teuthology.orchestra.run.smithi163.stderr:gzip -5 --verbose -- /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-volume.log 2024-06-18T03:19:05.522 INFO:teuthology.orchestra.run.smithi163.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/tcmu-runner.log: 87.8% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/tcmu-runner.log.gz 2024-06-18T03:19:05.527 INFO:teuthology.orchestra.run.smithi163.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-06-18T03:19:05.543 INFO:teuthology.orchestra.run.smithi163.stderr:/var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 90.9% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-volume.log.gz 2024-06-18T03:19:05.543 INFO:teuthology.orchestra.run.smithi186.stderr: 90.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-06-18T03:19:05.558 INFO:teuthology.orchestra.run.smithi163.stderr: 92.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-06-18T03:19:05.643 INFO:teuthology.orchestra.run.smithi186.stderr: 89.7% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mgr.x.log.gz 2024-06-18T03:19:05.843 INFO:teuthology.orchestra.run.smithi163.stderr: 89.2% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mgr.y.log.gz 2024-06-18T03:19:06.007 INFO:teuthology.orchestra.run.smithi186.stderr: 92.5% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mon.b.log.gz 2024-06-18T03:19:06.231 INFO:teuthology.orchestra.run.smithi163.stderr: 92.4% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mon.c.log.gz 2024-06-18T03:19:07.132 INFO:teuthology.orchestra.run.smithi163.stderr: 91.1% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-mon.a.log.gz 2024-06-18T03:19:07.138 INFO:teuthology.orchestra.run.smithi186.stderr: 93.2% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.6.log.gz 2024-06-18T03:19:07.300 INFO:teuthology.orchestra.run.smithi186.stderr: 93.4% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.5.log.gz 2024-06-18T03:19:07.340 INFO:teuthology.orchestra.run.smithi163.stderr: 93.0% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.2.log.gz 2024-06-18T03:19:07.345 INFO:teuthology.orchestra.run.smithi163.stderr: 93.4% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.0.log.gz 2024-06-18T03:19:07.473 INFO:teuthology.orchestra.run.smithi186.stderr: 93.8% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.7.log.gz 2024-06-18T03:19:07.485 INFO:teuthology.orchestra.run.smithi163.stderr: 93.5% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.1.log.gz 2024-06-18T03:19:07.614 INFO:teuthology.orchestra.run.smithi186.stderr: 93.6% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.4.log.gz 2024-06-18T03:19:07.617 INFO:teuthology.orchestra.run.smithi186.stderr: 2024-06-18T03:19:07.617 INFO:teuthology.orchestra.run.smithi186.stderr:real 0m2.137s 2024-06-18T03:19:07.617 INFO:teuthology.orchestra.run.smithi186.stderr:user 0m7.985s 2024-06-18T03:19:07.617 INFO:teuthology.orchestra.run.smithi186.stderr:sys 0m0.319s 2024-06-18T03:19:08.215 INFO:teuthology.orchestra.run.smithi163.stderr: 93.5% -- replaced with /var/log/ceph/929affee-2d1b-11ef-bc9d-c7b262605968/ceph-osd.3.log.gz 2024-06-18T03:19:08.217 INFO:teuthology.orchestra.run.smithi163.stderr: 2024-06-18T03:19:08.217 INFO:teuthology.orchestra.run.smithi163.stderr:real 0m2.739s 2024-06-18T03:19:08.217 INFO:teuthology.orchestra.run.smithi163.stderr:user 0m10.796s 2024-06-18T03:19:08.217 INFO:teuthology.orchestra.run.smithi163.stderr:sys 0m0.418s 2024-06-18T03:19:08.217 INFO:tasks.cephadm:Archiving logs... 2024-06-18T03:19:08.217 DEBUG:teuthology.misc:Transferring archived files from smithi163:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-06-17_21:08:03-orch-squid-distro-default-smithi/7760446/remote/smithi163/log 2024-06-18T03:19:08.219 DEBUG:teuthology.orchestra.run.smithi163:> sudo tar c -f - -C /var/log/ceph -- . 2024-06-18T03:19:09.720 DEBUG:teuthology.misc:Transferring archived files from smithi186:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-06-17_21:08:03-orch-squid-distro-default-smithi/7760446/remote/smithi186/log 2024-06-18T03:19:09.721 DEBUG:teuthology.orchestra.run.smithi186:> sudo tar c -f - -C /var/log/ceph -- . 2024-06-18T03:19:10.948 INFO:tasks.cephadm:Removing cluster... 2024-06-18T03:19:10.948 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 --force 2024-06-18T03:19:11.244 DEBUG:teuthology.orchestra.run.smithi186:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 929affee-2d1b-11ef-bc9d-c7b262605968 --force 2024-06-18T03:19:11.538 INFO:tasks.cephadm:Removing cephadm ... 2024-06-18T03:19:11.539 DEBUG:teuthology.orchestra.run.smithi163:> rm -rf /home/ubuntu/cephtest/cephadm 2024-06-18T03:19:11.554 DEBUG:teuthology.orchestra.run.smithi186:> rm -rf /home/ubuntu/cephtest/cephadm 2024-06-18T03:19:11.569 INFO:tasks.cephadm:Teardown complete 2024-06-18T03:19:11.570 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-06-18T03:19:11.581 INFO:teuthology.task.clock:Checking final clock skew... 2024-06-18T03:19:11.581 DEBUG:teuthology.orchestra.run.smithi163:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-18T03:19:11.596 DEBUG:teuthology.orchestra.run.smithi186:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-18T03:19:11.608 INFO:teuthology.orchestra.run.smithi163.stderr:bash: line 1: ntpq: command not found 2024-06-18T03:19:11.612 INFO:teuthology.orchestra.run.smithi163.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-18T03:19:11.612 INFO:teuthology.orchestra.run.smithi163.stdout:=============================================================================== 2024-06-18T03:19:11.612 INFO:teuthology.orchestra.run.smithi163.stdout:^+ hv01.front.sepia.ceph.com 3 6 377 28 +249us[ +249us] +/- 40ms 2024-06-18T03:19:11.612 INFO:teuthology.orchestra.run.smithi163.stdout:^* hv02.front.sepia.ceph.com 3 8 377 32 +326us[ +376us] +/- 58ms 2024-06-18T03:19:11.612 INFO:teuthology.orchestra.run.smithi163.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 27 -1067us[-1067us] +/- 84ms 2024-06-18T03:19:11.612 INFO:teuthology.orchestra.run.smithi163.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-06-18T03:19:11.623 INFO:teuthology.orchestra.run.smithi186.stderr:bash: line 1: ntpq: command not found 2024-06-18T03:19:11.627 INFO:teuthology.orchestra.run.smithi186.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-18T03:19:11.627 INFO:teuthology.orchestra.run.smithi186.stdout:=============================================================================== 2024-06-18T03:19:11.627 INFO:teuthology.orchestra.run.smithi186.stdout:^+ hv01.front.sepia.ceph.com 3 6 377 29 +285us[ +285us] +/- 40ms 2024-06-18T03:19:11.628 INFO:teuthology.orchestra.run.smithi186.stdout:^* hv02.front.sepia.ceph.com 3 6 377 35 +322us[ +382us] +/- 58ms 2024-06-18T03:19:11.628 INFO:teuthology.orchestra.run.smithi186.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 28 -1030us[-1030us] +/- 84ms 2024-06-18T03:19:11.628 INFO:teuthology.orchestra.run.smithi186.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-06-18T03:19:11.628 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-06-18T03:19:11.637 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-06-18T03:19:11.637 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-06-18T03:19:11.665 DEBUG:teuthology.orchestra.run.smithi163:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-06-18T03:19:11.750 DEBUG:teuthology.orchestra.run.smithi186:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-06-18T03:19:11.826 DEBUG:teuthology.orchestra.run.smithi163:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-06-18T03:19:11.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-18T03:19:11.856 DEBUG:teuthology.orchestra.run.smithi186:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-06-18T03:19:11.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-18T03:19:11.884 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-06-18T03:19:11.893 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-06-18T03:19:11.921 INFO:teuthology.task.internal:Duration was 2820.331593 seconds 2024-06-18T03:19:11.922 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-06-18T03:19:11.950 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-06-18T03:19:11.950 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-06-18T03:19:11.952 DEBUG:teuthology.orchestra.run.smithi186:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-06-18T03:19:11.989 INFO:teuthology.orchestra.run.smithi163.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-18T03:19:11.992 INFO:teuthology.orchestra.run.smithi186.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-18T03:19:12.327 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-06-18T03:19:12.327 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi163.front.sepia.ceph.com 2024-06-18T03:19:12.328 DEBUG:teuthology.orchestra.run.smithi163:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-06-18T03:19:12.387 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi186.front.sepia.ceph.com 2024-06-18T03:19:12.387 DEBUG:teuthology.orchestra.run.smithi186:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-06-18T03:19:12.412 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-06-18T03:19:12.412 DEBUG:teuthology.orchestra.run.smithi163:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-06-18T03:19:12.429 DEBUG:teuthology.orchestra.run.smithi186:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-06-18T03:19:12.485 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-06-18T03:19:12.485 DEBUG:teuthology.orchestra.run.smithi163:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-06-18T03:19:12.495 DEBUG:teuthology.orchestra.run.smithi186:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-06-18T03:19:13.080 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-06-18T03:19:13.090 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-06-18T03:19:13.090 DEBUG:teuthology.orchestra.run.smithi163:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-06-18T03:19:13.115 DEBUG:teuthology.orchestra.run.smithi186:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-06-18T03:19:13.140 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-06-18T03:19:13.149 DEBUG:teuthology.orchestra.run.smithi163:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-06-18T03:19:13.157 DEBUG:teuthology.orchestra.run.smithi186:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-06-18T03:19:13.178 INFO:teuthology.orchestra.run.smithi163.stdout:kernel.core_pattern = core 2024-06-18T03:19:13.203 INFO:teuthology.orchestra.run.smithi186.stdout:kernel.core_pattern = core 2024-06-18T03:19:13.233 DEBUG:teuthology.orchestra.run.smithi163:> test -e /home/ubuntu/cephtest/archive/coredump 2024-06-18T03:19:13.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-18T03:19:13.263 DEBUG:teuthology.orchestra.run.smithi186:> test -e /home/ubuntu/cephtest/archive/coredump 2024-06-18T03:19:13.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-18T03:19:13.288 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-06-18T03:19:13.300 INFO:teuthology.task.internal:Transferring archived files... 2024-06-18T03:19:13.301 DEBUG:teuthology.misc:Transferring archived files from smithi163:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-06-17_21:08:03-orch-squid-distro-default-smithi/7760446/remote/smithi163 2024-06-18T03:19:13.301 DEBUG:teuthology.orchestra.run.smithi163:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-06-18T03:19:13.358 DEBUG:teuthology.misc:Transferring archived files from smithi186:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-06-17_21:08:03-orch-squid-distro-default-smithi/7760446/remote/smithi186 2024-06-18T03:19:13.359 DEBUG:teuthology.orchestra.run.smithi186:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-06-18T03:19:13.400 INFO:teuthology.task.internal:Removing archive directory... 2024-06-18T03:19:13.400 DEBUG:teuthology.orchestra.run.smithi163:> rm -rf -- /home/ubuntu/cephtest/archive 2024-06-18T03:19:13.402 DEBUG:teuthology.orchestra.run.smithi186:> rm -rf -- /home/ubuntu/cephtest/archive 2024-06-18T03:19:13.447 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-06-18T03:19:13.458 INFO:teuthology.task.internal:Not uploading archives. 2024-06-18T03:19:13.459 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-06-18T03:19:13.483 INFO:teuthology.task.internal:Tidying up after the test... 2024-06-18T03:19:13.483 DEBUG:teuthology.orchestra.run.smithi163:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-06-18T03:19:13.486 DEBUG:teuthology.orchestra.run.smithi186:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-06-18T03:19:13.498 INFO:teuthology.orchestra.run.smithi163.stdout: 262165 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jun 18 03:19 /home/ubuntu/cephtest 2024-06-18T03:19:13.502 INFO:teuthology.orchestra.run.smithi186.stdout: 265313 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jun 18 03:19 /home/ubuntu/cephtest 2024-06-18T03:19:13.504 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-06-18T03:19:13.515 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-06-18T03:19:13.588 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 2820.3315925598145 failure_reason: '"2024-06-18T03:04:03.531297+0000 mon.a (mon.0) 1086 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log' owner: scheduled_teuthology@teuthology success: false 2024-06-18T03:19:13.589 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-06-18T03:19:13.686 INFO:teuthology.run:FAIL